diff --git a/dags/cumulus/aprfc_qpf_06h.py b/dags/cumulus/aprfc_qpf_06h.py index 7e57fb4..7f5b666 100644 --- a/dags/cumulus/aprfc_qpf_06h.py +++ b/dags/cumulus/aprfc_qpf_06h.py @@ -102,7 +102,7 @@ def get_filenames(edate, url): @dag( default_args=default_args, - schedule="20 9,15,19 * * *", + schedule="25 * * * *", tags=["cumulus", "precip", "QPF", "APRFC"], max_active_runs=1, max_active_tasks=1, @@ -126,10 +126,13 @@ def download_raw_qpf(): for filename in filenames: url = f"{URL_ROOT}/{filename}" s3_key = f"{key_prefix}/{PRODUCT_SLUG}/{filename}" - # Check if the file already exists in S3 - if s3_file_exists(cumulus.S3_BUCKET, s3_key): - print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") - continue # Skip to the next file + try: + # Check if the file already exists in S3 + if s3_file_exists(cumulus.S3_BUCKET, s3_key): + print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") + continue # Skip to the next file + except: + print(f'Error checking S3 for {s3_key}') print(f"Downloading file: {filename}") try: trigger_download(url=url, s3_bucket=cumulus.S3_BUCKET, s3_key=s3_key) diff --git a/dags/cumulus/aprfc_qtf_01h.py b/dags/cumulus/aprfc_qtf_01h.py index d6b8d7c..8dd0bc0 100644 --- a/dags/cumulus/aprfc_qtf_01h.py +++ b/dags/cumulus/aprfc_qtf_01h.py @@ -105,7 +105,7 @@ def get_filenames(edate, url): @dag( default_args=default_args, - schedule="21 9,15,19 * * *", + schedule="25 * * * *", tags=["cumulus", "temp", "QTF", "APRFC"], max_active_runs=1, max_active_tasks=1, @@ -129,10 +129,13 @@ def download_raw_qtf(): for filename in filenames: url = f"{URL_ROOT}/{filename}" s3_key = f"{key_prefix}/{PRODUCT_SLUG}/{filename}" - # Check if the file already exists in S3 - if s3_file_exists(cumulus.S3_BUCKET, s3_key): - print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") - continue # Skip to the next file + try: + # Check if the file already exists in S3 + if s3_file_exists(cumulus.S3_BUCKET, s3_key): + print(f"Skipping existing S3 object: s3://{cumulus.S3_BUCKET}/{s3_key}") + continue # Skip to the next file + except: + print(f'Error checking S3 for {s3_key}') print(f"Downloading file: {filename}") try: trigger_download(url=url, s3_bucket=cumulus.S3_BUCKET, s3_key=s3_key)