Housekeeping and Testing

.github/workflows/tests.yml

@@ -0,0 +1,59 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.11"]
+        include:
+        - os: ubuntu-latest
+          path: ~/.cache/pip
+        - os: macos-latest
+          path: ~/Library/Caches/pip
+        - os: windows-latest
+          path: ~\AppData\Local\pip\Cache
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip
+      uses: actions/cache@v4
+      with:
+        path: ${{ matrix.path }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -r requirements.txt
+        python -m pip install flake8==7.1.1 mypy==1.11.2 mypy-extensions types-pytz types-requests pytest==8.3.3 pytest-cov==5.0.0
+
+    - name: Lint with flake8
+      run: |
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --exit-zero --ignore=W191,E501 --max-complexity=25 --statistics
+
+    - name: Static type checking with mypy
+      run: |
+        python -m mypy --strict .
+
+    - name: Run Pytest
+      run: |
+        pytest
+
+    - name: Run tool with --help
+      run: |
+        python threat_intelligence_toolkit.py --help

.gitignore

@@ -0,0 +1,142 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.idea/
+
+docs/_build

README.md

@@ -4,7 +4,7 @@
 Automate generating or pulling threat intelligence Structured Threat Information Expression (STIX) files from a flat file or from a TAXII server and uploading a threat collection to an ECA and multiple EDAs via the REST API. By uploading STIX files, you can add a threat collection to your ExtraHop Discover and Command appliances. Threat collections enable you to identify suspicious hosts, IP addresses, and URIs on your network.
 
 ## Usage
-Specify an output directory, threat collection name, ECA/EDA details, and other optional config via the command line then the script either generates a STIX file or polls the TAXII server (defaults to [EclecticIQ OpenTAXII](https://open.taxiistand.com) intel feed), saves the stix files in a gzipped tar file (tgz), and uploads the threat collection to the specified Reveal(x) ECA/EDAs.
+Specify an output directory, threat collection name, ECA/EDA details, and other optional config via the command line then the script either generates a STIX file or polls the TAXII server (defaults to OTX AlienVault/AT&T Cybersecurity](https://otx.alienvault.com) intel feed), saves the stix files in a gzipped tar file (tgz), and uploads the threat collection to the specified Reveal(x) ECA/EDAs.
 
 **This script solely serves as example code and is made available without any support or warranty.**
 
@@ -21,16 +21,16 @@ You will need to update the example output paths, collection names, IP addresses
 - View usage and all possible command line arguments
   - `python3 threat_intelligence_toolkit.py -h`
 
-- Download, tgz, and upload all collections from the default EclecticIQ OpenTAXII server to an EDA
+- Download, tgz, and upload all collections from the default OTX AlienVault/AT&T Cybersecurity server to an EDA
   - `python3 threat_intelligence_toolkit.py -o ~/output_folder -tc example_collection --eda 172.16.1.2 3Hb7EpHRqb2EpnS7iweHgR5F3sf False`
 
-- Download, tgz, and upload all collections from the default EclecticIQ OpenTAXII server to an ECA and multiple EDAs
+- Download, tgz, and upload all collections from the default OTX AlienVault/AT&T Cybersecurity server to an ECA and multiple EDAs
   - `python3 threat_intelligence_toolkit.py -o ~/output_folder -tc example_collection --eca 172.16.1.1 3Hb7EpHRqb2EpnS7iweHgR5F3sg True --eda 172.16.1.2 3Hb7EpHRqb2EpnS7iweHgR5F3sf False --eda 172.16.1.3 3Hb7EpHRqb2EpnS7iweHgR5F3sf False`
 
-- Download and tgz all collections from the default EclecticIQ OpenTAXII server to be uploaded manually
+- Download and tgz all collections from the default OTX AlienVault/AT&T Cybersecurity server to be uploaded manually
   - `python3 threat_intelligence_toolkit.py -o ~/output_folder -tc example_collection`
 
-- Download and tgz a specific list of collections from the default EclecticIQ OpenTAXII server to be uploaded manually
+- Download and tgz a specific list of collections from the default OTX AlienVault/AT&T Cybersecurity server to be uploaded manually
   - `python3 threat_intelligence_toolkit.py -o ~/output_folder -tc example_collection --taxii-collections vxvault hailataxii.guest.dataForLast_7daysOnly`
 
 - Download, tgz, and upload all collections from a specific TAXII server to an EDA

mypy.ini

@@ -0,0 +1,6 @@
+# Global options:
+
+[mypy]
+warn_return_any = True
+warn_unused_configs = True
+ignore_missing_imports = True

pytest.ini

@@ -0,0 +1,11 @@
+# pytest.ini
+[pytest]
+pythonpath = .
+addopts = --cov=./ --cov-report=term-missing --cov-config=pytest.ini
+filterwarnings =
+        ignore::DeprecationWarning:libtaxii.*
+
+[coverage:run]
+omit =
+    */tests/*
+    */migrations/*

requirements.txt

@@ -1,21 +1,22 @@
 cabby==0.1.23
-certifi==2024.7.4
-charset-normalizer==2.1.1
-colorlog==6.7.0
+certifi==2024.8.30
+charset-normalizer==3.3.2
+colorlog==6.8.2
 cybox==2.1.0.21
 decorator==5.1.1
 furl==2.1.3
-idna==3.7
+idna==3.10
 libtaxii==1.1.119
-lxml==4.9.2
+lxml==5.3.0
 mixbox==1.0.5
 ordered-set==4.1.0
 orderedmultidict==1.0.1
-python-dateutil==2.8.2
-pytz==2022.6
-requests==2.32.0
+python-dateutil==2.9.0.post0
+pytz==2024.2
+requests==2.32.3
 six==1.16.0
 stix==1.2.0.11
-urllib3==1.26.19
-validators==0.20.0
+urllib3==2.2.3
+validators==0.34.0
 weakrefmethod==1.0.3
+