diff --git a/dags/cumulus/aprfc_qpf_06h.py b/dags/cumulus/aprfc_qpf_06h.py index 7f5b666..12712e5 100644 --- a/dags/cumulus/aprfc_qpf_06h.py +++ b/dags/cumulus/aprfc_qpf_06h.py @@ -106,6 +106,7 @@ def get_filenames(edate, url): tags=["cumulus", "precip", "QPF", "APRFC"], max_active_runs=1, max_active_tasks=1, + catchup=False, # Disable backfills ) def cumulus_aprfc_qpf_06h(): """This pipeline handles download, processing, and derivative product creation for \n @@ -126,13 +127,6 @@ def download_raw_qpf(): for filename in filenames: url = f"{URL_ROOT}/{filename}" s3_key = f"{key_prefix}/{PRODUCT_SLUG}/{filename}" - try: - # Check if the file already exists in S3 - if s3_file_exists(cumulus.S3_BUCKET, s3_key): - print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") - continue # Skip to the next file - except: - print(f'Error checking S3 for {s3_key}') print(f"Downloading file: {filename}") try: trigger_download(url=url, s3_bucket=cumulus.S3_BUCKET, s3_key=s3_key) diff --git a/dags/cumulus/aprfc_qtf_01h.py b/dags/cumulus/aprfc_qtf_01h.py index 8dd0bc0..0a6e4e5 100644 --- a/dags/cumulus/aprfc_qtf_01h.py +++ b/dags/cumulus/aprfc_qtf_01h.py @@ -109,6 +109,7 @@ def get_filenames(edate, url): tags=["cumulus", "temp", "QTF", "APRFC"], max_active_runs=1, max_active_tasks=1, + catchup=False, # Disable backfills ) def cumulus_aprfc_qtf_01h(): """This pipeline handles download, processing, and derivative product creation for \n @@ -129,13 +130,6 @@ def download_raw_qtf(): for filename in filenames: url = f"{URL_ROOT}/{filename}" s3_key = f"{key_prefix}/{PRODUCT_SLUG}/{filename}" - try: - # Check if the file already exists in S3 - if s3_file_exists(cumulus.S3_BUCKET, s3_key): - print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") - continue # Skip to the next file - except: - print(f'Error checking S3 for {s3_key}') print(f"Downloading file: {filename}") try: trigger_download(url=url, s3_bucket=cumulus.S3_BUCKET, s3_key=s3_key)