@@ -510,15 +510,19 @@ def render_main_page():
510510 )
511511
512512
513- def add_base_tag (html_content , original_url ):
513+ def process_html_document (html_content , original_url ):
514+ """
515+ Parses HTML, injects the <base> tag to fix relative links, and applies
516+ site-specific modifications (like stripping paywall JS) where needed.
517+ """
514518 soup = BeautifulSoup (html_content , "html.parser" )
515519 parsed_url = urlparse (original_url )
516520 base_url = f"{ parsed_url .scheme } ://{ parsed_url .netloc } /"
517521
518-
519522 # Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
520523 if parsed_url .path and not parsed_url .path .endswith ("/" ):
521524 base_url = urljoin (base_url , parsed_url .path .rsplit ("/" , 1 )[0 ] + "/" )
525+
522526 base_tag = soup .find ("base" )
523527 if not base_tag :
524528 new_base_tag = soup .new_tag ("base" , href = base_url )
@@ -529,6 +533,23 @@ def add_base_tag(html_content, original_url):
529533 head_tag .insert (0 , new_base_tag )
530534 soup .insert (0 , head_tag )
531535
536+ domain = parsed_url .netloc .lower ()
537+
538+ # --- SITE SPECIFIC FIXES ---
539+
540+ # The Seattle Times
541+ if "seattletimes.com" in domain :
542+ # Seattle Times uses specific JS bundles to trigger the paywall modal and ads.
543+ # Removing these scripts prevents the paywall from locking the screen.
544+ for script in soup .find_all ("script" , src = True ):
545+ if "st-user-messaging" in script ["src" ] or "st-advertising" in script ["src" ]:
546+ script .decompose ()
547+
548+ # Ensure the body isn't locked from scrolling by JS-injected inline styles
549+ if soup .body :
550+ current_style = soup .body .get ("style" , "" )
551+ soup .body ["style" ] = f"{ current_style } ; overflow: auto !important; position: static !important;"
552+
532553 return str (soup )
533554
534555
@@ -678,7 +699,7 @@ def bypass_paywall(url, strings, job_id=None):
678699 freedium_html , freedium_final = fetch_via_freedium (url , job_id )
679700 if freedium_html :
680701 set_step (job_id , 'process' )
681- result = add_base_tag (freedium_html , freedium_final )
702+ result = process_html_document (freedium_html , freedium_final )
682703 set_step (job_id , 'cleanup' )
683704 return result
684705 else :
@@ -713,7 +734,7 @@ def bypass_paywall(url, strings, job_id=None):
713734 raise UserFacingError (msg )
714735
715736 set_step (job_id , 'process' )
716- result = add_base_tag (html_text , final_url )
737+ result = process_html_document (html_text , final_url )
717738
718739 set_step (job_id , 'cleanup' )
719740 return result
0 commit comments