Skip to content

[IDEV-2271] RE:RTUF stream responses: address PR feedbacks #181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
briluza merged 6 commits into from
Oct 27, 2025
Merged

[IDEV-2271] RE:RTUF stream responses: address PR feedbacks #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
57afec8
address PR feedbacks
briluza Oct 23, 2025
cae89ce
removed condition in yielding line
briluza Oct 23, 2025
5254e00
remove unused imports; modify docstring for FeedsResult
briluza Oct 23, 2025
3a598fd
Remove x-api-key in test cassetes, disable rate limiting for rttf pro…
briluza Oct 23, 2025
6632898
address PR feedback
briluza Oct 27, 2025
7e981f0
remove _get_results and directly call the make_request
briluza Oct 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions domaintools/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,12 @@ def _build_api_url(self, api_url=None, api_port=None):

self._rest_api_url = rest_api_url

def _rate_limit(self):
def _rate_limit(self, product):
"""Pulls in and enforces the latest rate limits for the specified user"""
if product in RTTF_PRODUCTS_LIST:
self.limits_set = False
return

self.limits_set = True
for product in self.account_information():
limit_minutes = product["per_minute_limit"] or None
Expand All @@ -130,7 +134,7 @@ def _results(self, product, path, cls=Results, **kwargs):
if product != "account-information" and self.rate_limit and not self.limits_set and not self.limits:
always_sign_api_key_previous_value = self.always_sign_api_key
header_authentication_previous_value = self.header_authentication
self._rate_limit()
self._rate_limit(product)
# Reset always_sign_api_key and header_authentication to its original
# User-set values as these might be affected when self.account_information() was executed
self.always_sign_api_key = always_sign_api_key_previous_value
Expand Down
112 changes: 36 additions & 76 deletions domaintools/results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,22 @@
"""Defines the used Result object based on the current versions and/or features available to Python runtime

"""
Defines the used Result object based on the current versions and/or features available to Python runtime
Additionally, defines any custom result objects that may be used to enable more Pythonic interaction with endpoints.
"""

import json
import logging
from itertools import zip_longest, chain
from typing import Generator

import httpx
import time

try: # pragma: no cover
from collections import OrderedDict
except ImportError: # pragma: no cover
from ordereddict import OrderedDict

from itertools import zip_longest, chain
from typing import Generator
from httpx import Client

from domaintools_async import AsyncResults as Results


log = logging.getLogger(__name__)


Expand Down Expand Up @@ -210,13 +207,11 @@ class FeedsResults(Results):

Highlevel process:

httpx stream -> yield each json line -> check status code -> yield back data to client
httpx stream -> check status code -> yield back data to client -> repeat if 206

Returns the generator object for feeds results.
"""

latest_feeds_status_code = None

def _make_request(self) -> Generator:
"""
Creates and manages the httpx stream request, yielding data line by line.
Expand All @@ -226,76 +221,41 @@ def _make_request(self) -> Generator:
headers = session_info.get("headers")
parameters = session_info.get("parameters")

try:
with httpx.stream(
"GET",
self.url,
headers=headers,
params=parameters,
verify=self.api.verify_ssl,
proxy=self.api.proxy_url,
timeout=None,
) as response:
self.latest_feeds_status_code = response.status_code
# assigned the status code to `latest_feeds_status_code`
# started the process
yield {"status_ready": True}

for line in response.iter_lines():
if line:
yield line
except Exception as e:
self.latest_feeds_status_code = 500
yield {"status_ready": True, "error": str(e)}
raise

def _get_results(self) -> Generator:
try:
wait_for = self._wait_time()
feeds_generator = self._make_request()

next(feeds_generator) # to start the generator process
self.setStatus(self.latest_feeds_status_code) # set the status already

should_wait = (
wait_for
and wait_for > 0
and not (self.api.rate_limit and (self.product == "account-information"))
)
if should_wait:
log.info(f"Sleeping for {wait_for}s.")
time.sleep(wait_for)

# yield the rest of the feeds
yield from feeds_generator
except Exception as e:
log.error(f"FATAL: Failed to start the feed generator in _make_request. Reason: {e}")
Copy link

@kelvinatorr kelvinatorr Oct 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you want to just raise an Exception for the user you don't need to layer the try: except here and in the other functions. You can just remove all the try: except:'s and let the exception happen with the error text.

For example if the user has a bad value with after=abc here is what the error will look like:

(venv) (IDEV-2271)~/github.com/python_api$ python tester.py 
Traceback (most recent call last):
  File "/home/kdealca/github.com/python_api/domaintools/results.py", line 252, in _get_results
    self.setStatus(self.latest_feeds_status_code)  # set the status already
  File "/home/kdealca/github.com/python_api/domaintools/base_results.py", line 201, in setStatus
    raise BadRequestException(code, reason)
domaintools.exceptions.BadRequestException: None

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/kdealca/github.com/python_api/domaintools/results.py", line 270, in response
    yield from feed_response_generator
  File "/home/kdealca/github.com/python_api/domaintools/results.py", line 257, in _get_results
    self.setStatus(500, reason_text=e)
  File "/home/kdealca/github.com/python_api/domaintools/base_results.py", line 207, in setStatus
    raise InternalServerErrorException(code, reason)
domaintools.exceptions.InternalServerErrorException: None

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/kdealca/github.com/python_api/tester.py", line 9, in <module>
    for r in results.response():
  File "/home/kdealca/github.com/python_api/domaintools/results.py", line 279, in response
    self.setStatus(500, reason_text=e)
  File "/home/kdealca/github.com/python_api/domaintools/base_results.py", line 207, in setStatus
    raise InternalServerErrorException(code, reason)
domaintools.exceptions.InternalServerErrorException: None
(venv) (IDEV-2271)~/github.com/python_api$

It is 3 exceptions deep which makes it difficult to understand. And the helpful text from the server:

(IDEV-2271)~/github.com/python_api$ curl -H "X-API-Key: $DT_API_KEY" "https://api.domaintools.com/v1/feed/nod?after=abc"
{"error":"Bad request: invalid value in after query parameter; see documentation for examples"}
(IDEV-2271)~/github.com/python_api$

Is nowhere to be seen.

Instead you can do something like this:

    def _make_request(self) -> Generator:
        """
        Creates and manages the httpx stream request, yielding data line by line.
        This is the core generator that communicates with the DT frontend API server.
        """
        session_info = self._get_session_params_and_headers()
        headers = session_info.get("headers")
        parameters = session_info.get("parameters")

        with httpx.stream(
            "GET",
            self.url,
            headers=headers,
            params=parameters,
            verify=self.api.verify_ssl,
            proxy=self.api.proxy_url,
            timeout=None,
        ) as r:
            error_text = ""
            if r.status_code != 200 or r.status_code != 206:
                r.read()
                error_text = r.text
            self.setStatus(r.status_code, reason_text=error_text)

            for line in r.iter_lines():
                yield line

    def data(self) -> Generator:
        self._data = self._get_results()
        return self._data

    def response(self) -> Generator:
        while self.status != 200:
            yield from self.data()
            if not self.kwargs.get("sessionID"):
                # we'll only do iterative request for queries that has sessionID. Otherwise, we will have an infinite
                # request if sessionID was not provided but the required data asked is more than the maximum (1 hour of data)
                break
        self._status = None

If there is an error there will only be one, and any message from the server will be shown:

(venv) (IDEV-2271)~/github.com/python_api$ python tester.py 
Traceback (most recent call last):
  File "/home/kdealca/github.com/python_api/tester.py", line 9, in <module>
    for r in results.response():
  File "/home/kdealca/github.com/python_api/domaintools/results.py", line 248, in response
    yield from self.data()
  File "/home/kdealca/github.com/python_api/domaintools/results.py", line 237, in _make_request
    self.setStatus(r.status_code, reason_text=error_text)
  File "/home/kdealca/github.com/python_api/domaintools/base_results.py", line 201, in setStatus
    raise BadRequestException(code, reason)
domaintools.exceptions.BadRequestException: {"error":"Bad request: invalid value in after query parameter; see documentation for examples"}

(venv) (IDEV-2271)~/github.com/python_api$

Copy link
Contributor Author

@briluza briluza Oct 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for the feedback @kelvinatorr! will do the corresponding changes

Copy link
Contributor Author

@briluza briluza Oct 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

self.latest_feeds_status_code = 500
self.setStatus(500, reason_text=f"Reason: {e}")
return
with httpx.stream(
"GET",
self.url,
headers=headers,
params=parameters,
verify=self.api.verify_ssl,
proxy=self.api.proxy_url,
timeout=None,
) as response:
# set the status already
error_text = ""
status_code = response.status_code
if status_code not in [200, 206]:
response.read()
error_text = response.text

self.setStatus(status_code, reason_text=error_text)

for line in response.iter_lines():
yield line

def data(self) -> Generator:
self._data = self._get_results()
self._data = self._make_request()
return self._data

def response(self) -> Generator:
status_code = None
while status_code != 200:
try:
feed_response_generator = self.data()

yield from feed_response_generator
status_code = self.status
self._data = None # clear the data here

if not self.kwargs.get("sessionID"):
# we'll only do iterative request for queries that has sessionID.
# Otherwise, we will have an infinite request if sessionID was not provided but the required data asked is more than the maximum (1 hour of data)
break
except Exception as e:
self.latest_feeds_status_code = 500
self.setStatus(500, reason_text=f"Reason: {e}")
break # safely close the while loop if there's any error above
while self.status != 200:
yield from self.data()

if not self.kwargs.get("sessionID"):
# we'll only do iterative request for queries that has sessionID.
# Otherwise, we will have an infinite request if sessionID was not provided
# but the required data asked is more than the maximum (1 hour of data)
break
self._status = None

def __str__(self):
return f"{self.__class__.__name__} - {self.product}"
138 changes: 11 additions & 127 deletions tests/fixtures/vcr/test_domain_discovery_feed.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2990,162 +2990,46 @@ interactions:
- api.domaintools.com
user-agent:
- python-httpx/0.28.1
x-api-key:
- 4b02d-a4719-e33e7-93128-5a5ff
method: GET
uri: https://api.domaintools.com/v1/feed/domaindiscovery/?after=-60&app_name=python_wrapper&app_version=2.5.2&header_authentication=false
response:
body:
string: '{"timestamp":"2025-10-06T16:40:38Z","domain":"albrologistics.com"}

{"timestamp":"2025-10-06T16:40:39Z","domain":"datablog.co.uk"}

{"timestamp":"2025-10-06T16:40:39Z","domain":"stelapratas.com"}

{"timestamp":"2025-10-06T16:40:39Z","domain":"donasveganasymas.com"}

{"timestamp":"2025-10-06T16:40:39Z","domain":"rdstore.shop"}

{"timestamp":"2025-10-06T16:40:39Z","domain":"basopbafep.at"}

'
headers:
Cache-Control:
- no-store, no-cache, must-revalidate
Content-Security-Policy:
- 'default-src * data: blob: ''unsafe-eval'' ''unsafe-inline'''
Content-Type:
- application/x-ndjson
Date:
- Mon, 06 Oct 2025 16:41:37 GMT
Expires:
- Thu, 19 Nov 1981 08:52:00 GMT
Pragma:
- no-cache
Set-Cookie:
- dtsession=mjnphs9kpucpipg5t5mksdvmchdrtomgf49bb5mnu64c971099658j14c65q5jhrh2f9pu589th4rt89d7lj2jq96p71d9m0fd854dm;
expires=Wed, 05-Nov-2025 16:41:37 GMT; Max-Age=2592000; path=/; domain=.domaintools.com;
secure; HttpOnly
- 0566fae076d9b6615cd8f47a9e9500f2=da9b98176bbb72d0ca7886ae363d40d4; path=/;
HttpOnly
Strict-Transport-Security:
- max-age=31536000; includeSubDomains
Transfer-Encoding:
- chunked
status:
code: 200
message: OK
- request:
body: ''
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate
connection:
- keep-alive
host:
- api.domaintools.com
user-agent:
- python-httpx/0.28.1
x-api-key:
- 4b02d-a4719-e33e7-93128-5a5ff
method: GET
uri: https://api.domaintools.com/v1/feed/domaindiscovery/?after=-60&app_name=python_wrapper&app_version=2.5.3&top=5
response:
body:
string: '{"timestamp":"2025-10-08T13:34:20Z","domain":"ordalab.com"}

{"timestamp":"2025-10-08T13:34:20Z","domain":"shopwert.online"}

{"timestamp":"2025-10-08T13:34:20Z","domain":"ryaparts.fr"}

{"timestamp":"2025-10-08T13:34:20Z","domain":"21637.cn"}

{"timestamp":"2025-10-08T13:34:19Z","domain":"15054.cn"}

'
headers:
Cache-Control:
- no-store, no-cache, must-revalidate
Content-Length:
- '298'
Content-Security-Policy:
- 'default-src * data: blob: ''unsafe-eval'' ''unsafe-inline'''
Content-Type:
- application/x-ndjson
Date:
- Wed, 08 Oct 2025 13:34:21 GMT
Expires:
- Thu, 19 Nov 1981 08:52:00 GMT
Pragma:
- no-cache
Set-Cookie:
- dtsession=osvoe5vbplha77upcr3sdbhdchlemr0uacv0ar7rp6uecprl0hbcq89ooqvh6osi5ud5blhpl367lvlunomgar40tknvp4i8h9s66dr;
expires=Fri, 07-Nov-2025 13:34:21 GMT; Max-Age=2592000; path=/; domain=.domaintools.com;
secure; HttpOnly
- 0566fae076d9b6615cd8f47a9e9500f2=0335edaf1718dccec7bc612e6dd71941; path=/;
HttpOnly
Strict-Transport-Security:
- max-age=31536000; includeSubDomains
status:
code: 200
message: OK
- request:
body: ''
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate
connection:
- keep-alive
host:
- api.domaintools.com
user-agent:
- python-httpx/0.28.1
x-api-key:
- 4b02d-a4719-e33e7-93128-5a5ff
method: GET
uri: https://api.domaintools.com/v1/feed/domaindiscovery/?after=-60&app_name=python_wrapper&app_version=2.6.0&top=5
response:
body:
string: '{"timestamp":"2025-10-22T10:52:26Z","domain":"campredemption.com"}
string: '{"timestamp":"2025-10-23T18:32:45Z","domain":"lahntonaci.de"}

{"timestamp":"2025-10-22T10:52:26Z","domain":"mahkota77.sbs"}
{"timestamp":"2025-10-23T18:32:45Z","domain":"wildeland-ventures.com"}

{"timestamp":"2025-10-22T10:52:25Z","domain":"kylothedogebnb.fun"}
{"timestamp":"2025-10-23T18:32:45Z","domain":"elartesanal.com.gt"}

{"timestamp":"2025-10-22T10:52:25Z","domain":"schwab-secure93.wiki"}
{"timestamp":"2025-10-23T18:32:45Z","domain":"5movierulz2.blue"}

{"timestamp":"2025-10-22T10:52:25Z","domain":"ok08.cc"}
{"timestamp":"2025-10-23T18:32:45Z","domain":"airplanepilotsupport.com"}

'
headers:
Cache-Control:
- no-store, no-cache, must-revalidate
Content-Length:
- '321'
- '338'
Content-Security-Policy:
- 'default-src * data: blob: ''unsafe-eval'' ''unsafe-inline'''
Content-Type:
- application/x-ndjson
Date:
- Wed, 22 Oct 2025 10:52:27 GMT
- Thu, 23 Oct 2025 18:32:45 GMT
Expires:
- Thu, 19 Nov 1981 08:52:00 GMT
Pragma:
- no-cache
Set-Cookie:
- dtsession=nd7k8vjf050qnsmprlcu9iedq156h2slia8fg4qa2d72h2bsfrvgd22qufudnh9u02233jg6k0k3lrmec2klf0rhg8ingdt9v2vs7fr;
expires=Fri, 21-Nov-2025 10:52:27 GMT; Max-Age=2592000; path=/; domain=.domaintools.com;
- dtsession=qe90675g75jckqeks6pbr4su7h2i0iul83l3r7gqcoi7jmlt6cj909gha30fmcajg3avge6ao00usmpdd7h6lsvqe22j6cv4n0vemts;
expires=Sat, 22-Nov-2025 18:32:45 GMT; Max-Age=2592000; path=/; domain=.domaintools.com;
secure; HttpOnly
- 0566fae076d9b6615cd8f47a9e9500f2=acfc58822336225f812eb799691c0d9e; path=/;
- 0566fae076d9b6615cd8f47a9e9500f2=4556b46b52fb2ae62de3b26ddc27dfa0; path=/;
HttpOnly
Strict-Transport-Security:
- max-age=31536000; includeSubDomains
X-Request-Id:
- a88466ad-c0e4-420c-ae51-7aebc8c97deb
- 4fbb1620-8c58-49f3-8f25-9523323808f8
status:
code: 200
message: OK
Expand Down
Loading