Skip to content

Commit e513a8d

Browse files
committed
Fix for seattle times
Closes #34
1 parent 0df86cf commit e513a8d

1 file changed

Lines changed: 25 additions & 4 deletions

File tree

app/portable.py

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -510,15 +510,19 @@ def render_main_page():
510510
)
511511

512512

513-
def add_base_tag(html_content, original_url):
513+
def process_html_document(html_content, original_url):
514+
"""
515+
Parses HTML, injects the <base> tag to fix relative links, and applies
516+
site-specific modifications (like stripping paywall JS) where needed.
517+
"""
514518
soup = BeautifulSoup(html_content, "html.parser")
515519
parsed_url = urlparse(original_url)
516520
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
517521

518-
519522
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
520523
if parsed_url.path and not parsed_url.path.endswith("/"):
521524
base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
525+
522526
base_tag = soup.find("base")
523527
if not base_tag:
524528
new_base_tag = soup.new_tag("base", href=base_url)
@@ -529,6 +533,23 @@ def add_base_tag(html_content, original_url):
529533
head_tag.insert(0, new_base_tag)
530534
soup.insert(0, head_tag)
531535

536+
domain = parsed_url.netloc.lower()
537+
538+
# --- SITE SPECIFIC FIXES ---
539+
540+
# The Seattle Times
541+
if "seattletimes.com" in domain:
542+
# Seattle Times uses specific JS bundles to trigger the paywall modal and ads.
543+
# Removing these scripts prevents the paywall from locking the screen.
544+
for script in soup.find_all("script", src=True):
545+
if "st-user-messaging" in script["src"] or "st-advertising" in script["src"]:
546+
script.decompose()
547+
548+
# Ensure the body isn't locked from scrolling by JS-injected inline styles
549+
if soup.body:
550+
current_style = soup.body.get("style", "")
551+
soup.body["style"] = f"{current_style}; overflow: auto !important; position: static !important;"
552+
532553
return str(soup)
533554

534555

@@ -678,7 +699,7 @@ def bypass_paywall(url, strings, job_id=None):
678699
freedium_html, freedium_final = fetch_via_freedium(url, job_id)
679700
if freedium_html:
680701
set_step(job_id, 'process')
681-
result = add_base_tag(freedium_html, freedium_final)
702+
result = process_html_document(freedium_html, freedium_final)
682703
set_step(job_id, 'cleanup')
683704
return result
684705
else:
@@ -713,7 +734,7 @@ def bypass_paywall(url, strings, job_id=None):
713734
raise UserFacingError(msg)
714735

715736
set_step(job_id, 'process')
716-
result = add_base_tag(html_text, final_url)
737+
result = process_html_document(html_text, final_url)
717738

718739
set_step(job_id, 'cleanup')
719740
return result

0 commit comments

Comments
 (0)