Merge pull request #2352 from blacklanternsecurity/bump-httpx-0-28

Bumping httpx / pytest-httpx versions

Author: TheTechromancer

bbot/core/helpers/misc.py

@@ -50,7 +50,7 @@ def is_domain(d):
     if is_ip(d):
         return False
     extracted = tldextract(d)
-    if extracted.registered_domain:
+    if extracted.top_domain_under_public_suffix:
         if not extracted.subdomain:
             return True
     else:
@@ -85,7 +85,7 @@ def is_subdomain(d):
     if is_ip(d):
         return False
     extracted = tldextract(d)
-    if extracted.registered_domain:
+    if extracted.top_domain_under_public_suffix:
         if extracted.subdomain:
             return True
     else:
@@ -486,7 +486,7 @@ def split_domain(hostname):
         return ("", hostname)
     parsed = tldextract(hostname)
     subdomain = parsed.subdomain
-    domain = parsed.registered_domain
+    domain = parsed.top_domain_under_public_suffix
     if not domain:
         split = hostname.split(".")
         subdomain = ".".join(split[:-2])

bbot/core/helpers/web/client.py

@@ -70,7 +70,7 @@ def __init__(self, *args, **kwargs):
         kwargs["headers"] = headers
         # proxy
         proxies = self._web_config.get("http_proxy", None)
-        kwargs["proxies"] = proxies
+        kwargs["proxy"] = proxies
 
         log.verbose(f"Creating httpx.AsyncClient({args}, {kwargs})")
         super().__init__(*args, **kwargs)

bbot/core/helpers/web/web.py

@@ -95,7 +95,7 @@ async def request(self, *args, **kwargs):
             files (dict, optional): Dictionary of 'name': file-like-objects for multipart encoding upload.
             auth (tuple, optional): Auth tuple to enable Basic/Digest/Custom HTTP auth.
             timeout (float, optional): The maximum time to wait for the request to complete.
-            proxies (dict, optional): Dictionary mapping protocol schemes to proxy URLs.
+            proxy (str, optional): HTTP proxy URL.
             allow_redirects (bool, optional): Enables or disables redirection. Defaults to None.
             stream (bool, optional): Enables or disables response streaming.
             raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False.

bbot/modules/baddns.py

@@ -22,7 +22,7 @@ class baddns(BaseModule):
         "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     def select_modules(self):
         selected_submodules = []

bbot/modules/baddns_direct.py

@@ -19,7 +19,7 @@ class baddns_direct(BaseModule):
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     scope_distance_modifier = 1
 

bbot/modules/baddns_zone.py

@@ -16,7 +16,7 @@ class baddns_zone(baddns_module):
         "only_high_confidence": "Do not emit low-confidence or generic detections",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     def set_modules(self):
         self.enabled_submodules = ["NSEC", "zonetransfer"]

bbot/modules/internal/speculate.py

@@ -176,8 +176,8 @@ async def handle_event(self, event):
         org_stubs = set()
         if event.type == "DNS_NAME" and event.scope_distance == 0:
             tldextracted = self.helpers.tldextract(event.data)
-            registered_domain = getattr(tldextracted, "registered_domain", "")
-            if registered_domain:
+            top_domain_under_public_suffix = getattr(tldextracted, "top_domain_under_public_suffix", "")
+            if top_domain_under_public_suffix:
                 tld_stub = getattr(tldextracted, "domain", "")
                 if tld_stub:
                     decoded_tld_stub = self.helpers.smart_decode_punycode(tld_stub)

bbot/test/test_step_2/module_tests/test_module_censys.py

@@ -7,7 +7,7 @@ class TestCensys(ModuleTestBase):
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v1/account",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
             json={
                 "email": "[email protected]",
                 "login": "nope",
@@ -18,8 +18,9 @@ async def setup_before_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v2/certificates/search",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            match_content=b'{"q": "names: blacklanternsecurity.com", "per_page": 100}',
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            method="POST",
+            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100},
             json={
                 "code": 200,
                 "status": "OK",
@@ -47,8 +48,9 @@ async def setup_before_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v2/certificates/search",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            match_content=b'{"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"}',
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            method="POST",
+            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"},
             json={
                 "code": 200,
                 "status": "OK",

bbot/test/test_step_2/module_tests/test_module_ipstack.py

@@ -26,7 +26,7 @@ async def setup_before_prep(self, module_test):
                     "capital": "Washington D.C.",
                     "languages": [{"code": "en", "name": "English", "native": "English"}],
                     "country_flag": "https://assets.ipstack.com/flags/us.svg",
-                    "country_flag_emoji": "\ud83c\uddfa\ud83c\uddf8",
+                    "country_flag_emoji": "🇺🇸",
                     "country_flag_emoji_unicode": "U+1F1FA U+1F1F8",
                     "calling_code": "1",
                     "is_eu": False,
@@ -53,7 +53,7 @@ async def setup_before_prep(self, module_test):
                     "capital": "Washington D.C.",
                     "languages": [{"code": "en", "name": "English", "native": "English"}],
                     "country_flag": "https://assets.ipstack.com/flags/us.svg",
-                    "country_flag_emoji": "\ud83c\uddfa\ud83c\uddf8",
+                    "country_flag_emoji": "🇺🇸",
                     "country_flag_emoji_unicode": "U+1F1FA U+1F1F8",
                     "calling_code": "1",
                     "is_eu": False,

bbot/test/test_step_2/module_tests/test_module_postman.py

@@ -45,7 +45,22 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 25, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}',
+            match_json={
+                "service": "search",
+                "method": "POST",
+                "path": "/search-all",
+                "body": {
+                    "queryIndices": ["collaboration.workspace"],
+                    "queryText": "blacklanternsecurity",
+                    "size": 25,
+                    "from": 0,
+                    "clientTraceId": "",
+                    "requestOrigin": "srp",
+                    "mergeEntities": "true",
+                    "nonNestedRequests": "true",
+                    "domain": "public",
+                },
+            },
             json={
                 "data": [
                     {
@@ -177,7 +192,11 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}',
+            match_json={
+                "service": "workspaces",
+                "method": "GET",
+                "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public",
+            },
             json={
                 "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
                 "data": [
@@ -207,7 +226,11 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=testteam&slug=testing-bbot-api"}',
+            match_json={
+                "service": "workspaces",
+                "method": "GET",
+                "path": "/workspaces?handle=testteam&slug=testing-bbot-api",
+            },
             json={
                 "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
                 "data": [