From 3eb7af4a5af717c3ff70801d2ca9db95ec0225f6 Mon Sep 17 00:00:00 2001 From: msweier Date: Mon, 27 Oct 2025 08:56:53 -0500 Subject: [PATCH] remove s3 check, disable backfill --- dags/cumulus/aprfc_qpf_06h.py | 8 +------- dags/cumulus/aprfc_qtf_01h.py | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/dags/cumulus/aprfc_qpf_06h.py b/dags/cumulus/aprfc_qpf_06h.py index 7f5b666..12712e5 100644 --- a/dags/cumulus/aprfc_qpf_06h.py +++ b/dags/cumulus/aprfc_qpf_06h.py @@ -106,6 +106,7 @@ def get_filenames(edate, url): tags=["cumulus", "precip", "QPF", "APRFC"], max_active_runs=1, max_active_tasks=1, + catchup=False, # Disable backfills ) def cumulus_aprfc_qpf_06h(): """This pipeline handles download, processing, and derivative product creation for \n @@ -126,13 +127,6 @@ def download_raw_qpf(): for filename in filenames: url = f"{URL_ROOT}/{filename}" s3_key = f"{key_prefix}/{PRODUCT_SLUG}/{filename}" - try: - # Check if the file already exists in S3 - if s3_file_exists(cumulus.S3_BUCKET, s3_key): - print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") - continue # Skip to the next file - except: - print(f'Error checking S3 for {s3_key}') print(f"Downloading file: {filename}") try: trigger_download(url=url, s3_bucket=cumulus.S3_BUCKET, s3_key=s3_key) diff --git a/dags/cumulus/aprfc_qtf_01h.py b/dags/cumulus/aprfc_qtf_01h.py index 8dd0bc0..0a6e4e5 100644 --- a/dags/cumulus/aprfc_qtf_01h.py +++ b/dags/cumulus/aprfc_qtf_01h.py @@ -109,6 +109,7 @@ def get_filenames(edate, url): tags=["cumulus", "temp", "QTF", "APRFC"], max_active_runs=1, max_active_tasks=1, + catchup=False, # Disable backfills ) def cumulus_aprfc_qtf_01h(): """This pipeline handles download, processing, and derivative product creation for \n @@ -129,13 +130,6 @@ def download_raw_qtf(): for filename in filenames: url = f"{URL_ROOT}/{filename}" s3_key = f"{key_prefix}/{PRODUCT_SLUG}/{filename}" - try: - # Check if the file already exists in S3 - if s3_file_exists(cumulus.S3_BUCKET, s3_key): - print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") - continue # Skip to the next file - except: - print(f'Error checking S3 for {s3_key}') print(f"Downloading file: {filename}") try: trigger_download(url=url, s3_bucket=cumulus.S3_BUCKET, s3_key=s3_key)