From 523060a25bd1320584deab3f3db28e9197161cac Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 09:24:34 -0700
Subject: [PATCH 1/7] Remove redundant dag_id index on log table
(cherry picked from commit 388b60fa3b745289f5ab7c8692752b40ebef32f8)
---
airflow/models/log.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/airflow/models/log.py b/airflow/models/log.py
index 145f659b338d..e01cc29b45e0 100644
--- a/airflow/models/log.py
+++ b/airflow/models/log.py
@@ -49,7 +49,6 @@ class Log(Base):
try_number = Column(Integer)
__table_args__ = (
- Index("idx_log_dag", dag_id),
Index("idx_log_dttm", dttm),
Index("idx_log_event", event),
Index("idx_log_task_instance", dag_id, task_id, run_id, map_index, try_number),
From 4d3ed110d71e610b4a1451650d3155431373d7e4 Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 09:40:25 -0700
Subject: [PATCH 2/7] add migration
---
airflow/utils/db.py | 2 +-
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 245 +++++++++++----------
docs/apache-airflow/migrations-ref.rst | 4 +-
4 files changed, 133 insertions(+), 120 deletions(-)
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 1dc7275610df..47fdb9970f54 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -90,13 +90,13 @@ class MappedClassProtocol(Protocol):
log = logging.getLogger(__name__)
_REVISION_HEADS_MAP: dict[str, str] = {
+ "3.0.0": "522625f6d606",
"2.7.0": "405de8318b3a",
"2.8.0": "10b52ebd31f7",
"2.8.1": "88344c1d9134",
"2.9.0": "1949afb29106",
"2.9.2": "686269002441",
"2.10.0": "22ed7efa9da2",
- "3.0.0": "522625f6d606",
}
diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256
index 94ef213472b8..c48a05b94722 100644
--- a/docs/apache-airflow/img/airflow_erd.sha256
+++ b/docs/apache-airflow/img/airflow_erd.sha256
@@ -1 +1 @@
-d58a76011bd215ac7a0c42146986ad0eead5f4a9c8ffd7932b1aa1eb61e6e4af
\ No newline at end of file
+b937654c9ed767d8086d74e155d5bfc29ca6f44f8fcd8e3496fa603f37527cd9
\ No newline at end of file
diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg
index 3eaf0f0e52ab..65f94c58ad24 100644
--- a/docs/apache-airflow/img/airflow_erd.svg
+++ b/docs/apache-airflow/img/airflow_erd.svg
@@ -64,131 +64,131 @@
[INTEGER]
-
-
-dag_priority_parsing_request
-
-dag_priority_parsing_request
-
-id
-
- [VARCHAR(32)]
- NOT NULL
-
-fileloc
-
- [VARCHAR(2000)]
- NOT NULL
-
-
+
job
-
-job
-
-id
-
- [INTEGER]
- NOT NULL
-
-dag_id
-
- [VARCHAR(250)]
-
-end_date
-
- [TIMESTAMP]
-
-executor_class
-
- [VARCHAR(500)]
-
-hostname
-
- [VARCHAR(500)]
-
-job_type
-
- [VARCHAR(30)]
-
-latest_heartbeat
-
- [TIMESTAMP]
-
-start_date
-
- [TIMESTAMP]
-
-state
-
- [VARCHAR(20)]
-
-unixname
-
- [VARCHAR(1000)]
+
+job
+
+id
+
+ [INTEGER]
+ NOT NULL
+
+dag_id
+
+ [VARCHAR(250)]
+
+end_date
+
+ [TIMESTAMP]
+
+executor_class
+
+ [VARCHAR(500)]
+
+hostname
+
+ [VARCHAR(500)]
+
+job_type
+
+ [VARCHAR(30)]
+
+latest_heartbeat
+
+ [TIMESTAMP]
+
+start_date
+
+ [TIMESTAMP]
+
+state
+
+ [VARCHAR(20)]
+
+unixname
+
+ [VARCHAR(1000)]
-
+
slot_pool
-
-slot_pool
-
-id
-
- [INTEGER]
- NOT NULL
-
-description
-
- [TEXT]
-
-include_deferred
-
- [BOOLEAN]
- NOT NULL
-
-pool
-
- [VARCHAR(256)]
-
-slots
-
- [INTEGER]
+
+slot_pool
+
+id
+
+ [INTEGER]
+ NOT NULL
+
+description
+
+ [TEXT]
+
+include_deferred
+
+ [BOOLEAN]
+ NOT NULL
+
+pool
+
+ [VARCHAR(256)]
+
+slots
+
+ [INTEGER]
-
+
callback_request
-
-callback_request
-
-id
-
- [INTEGER]
- NOT NULL
-
-callback_data
-
- [JSON]
- NOT NULL
-
-callback_type
-
- [VARCHAR(20)]
- NOT NULL
-
-created_at
-
- [TIMESTAMP]
- NOT NULL
-
-priority_weight
-
- [INTEGER]
- NOT NULL
-
-processor_subdir
-
- [VARCHAR(2000)]
+
+callback_request
+
+id
+
+ [INTEGER]
+ NOT NULL
+
+callback_data
+
+ [JSON]
+ NOT NULL
+
+callback_type
+
+ [VARCHAR(20)]
+ NOT NULL
+
+created_at
+
+ [TIMESTAMP]
+ NOT NULL
+
+priority_weight
+
+ [INTEGER]
+ NOT NULL
+
+processor_subdir
+
+ [VARCHAR(2000)]
+
+
+
+dag_priority_parsing_request
+
+dag_priority_parsing_request
+
+id
+
+ [VARCHAR(32)]
+ NOT NULL
+
+fileloc
+
+ [VARCHAR(2000)]
+ NOT NULL
@@ -2349,5 +2349,16 @@
0..N
{0,1}
+
+
+alembic_version_fab
+
+alembic_version_fab
+
+version_num
+
+ [VARCHAR(32)]
+ NOT NULL
+
diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst
index 65a07aaa4393..3884db0564c3 100644
--- a/docs/apache-airflow/migrations-ref.rst
+++ b/docs/apache-airflow/migrations-ref.rst
@@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| Revision ID | Revises ID | Airflow Version | Description |
+=========================+==================+===================+==============================================================+
-| ``522625f6d606`` (head) | ``1cdc775ca98f`` | ``3.0.0`` | Add tables for backfill. |
+| ``16cbcb1c8c36`` (head) | ``522625f6d606`` | ``3.0.0`` | Remove redundant index |
++-------------------------+------------------+-------------------+--------------------------------------------------------------+
+| ``522625f6d606`` | ``1cdc775ca98f`` | ``3.0.0`` | Add tables for backfill. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| ``1cdc775ca98f`` | ``a2c32e6c7729`` | ``3.0.0`` | Drop ``execution_date`` unique constraint on DagRun. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
From 7684ad18f02e3ca305280f8baf6334e55c879b10 Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 09:42:09 -0700
Subject: [PATCH 3/7] fixup! add migration
---
.../0033_3_0_0_remove_redundant_index.py | 55 +++++++++++++++++++
1 file changed, 55 insertions(+)
create mode 100644 airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
diff --git a/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
new file mode 100644
index 000000000000..fb4644ea216a
--- /dev/null
+++ b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Remove redundant index
+
+Revision ID: 16cbcb1c8c36
+Revises: 522625f6d606
+Create Date: 2024-09-20 09:32:48.707545
+
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "16cbcb1c8c36"
+down_revision = "522625f6d606"
+branch_labels = None
+depends_on = None
+airflow_version = "3.0.0"
+
+
+def upgrade():
+ """Apply Remove redundant index"""
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("log", schema=None) as batch_op:
+ batch_op.drop_index("idx_log_dag")
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ """Unapply Remove redundant index"""
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("log", schema=None) as batch_op:
+ batch_op.create_index("idx_log_dag", ["dag_id"], unique=False)
+
+ # ### end Alembic commands ###
From fd5ecb0e63417bcfb6425a61dafccd27b4945100 Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 09:43:35 -0700
Subject: [PATCH 4/7] fixup! fixup! add migration
---
airflow/utils/db.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 47fdb9970f54..b11e744198dc 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -90,13 +90,13 @@ class MappedClassProtocol(Protocol):
log = logging.getLogger(__name__)
_REVISION_HEADS_MAP: dict[str, str] = {
- "3.0.0": "522625f6d606",
"2.7.0": "405de8318b3a",
"2.8.0": "10b52ebd31f7",
"2.8.1": "88344c1d9134",
"2.9.0": "1949afb29106",
"2.9.2": "686269002441",
"2.10.0": "22ed7efa9da2",
+ "3.0.0": "16cbcb1c8c36",
}
From a40990959dd3bcdacda873bcadab808ff63765c0 Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 10:10:58 -0700
Subject: [PATCH 5/7] fix static
---
.../versions/0033_3_0_0_remove_redundant_index.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
index fb4644ea216a..5dab860dbaf8 100644
--- a/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
+++ b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
@@ -17,7 +17,7 @@
# under the License.
"""
-Remove redundant index
+Remove redundant index.
Revision ID: 16cbcb1c8c36
Revises: 522625f6d606
@@ -38,7 +38,7 @@
def upgrade():
- """Apply Remove redundant index"""
+ """Apply Remove redundant index."""
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("log", schema=None) as batch_op:
batch_op.drop_index("idx_log_dag")
@@ -47,7 +47,7 @@ def upgrade():
def downgrade():
- """Unapply Remove redundant index"""
+ """Unapply Remove redundant index."""
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("log", schema=None) as batch_op:
batch_op.create_index("idx_log_dag", ["dag_id"], unique=False)
From 97d37e5cda5b11dd99a0a2c8ca085feb2a4bd0f7 Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 10:23:51 -0700
Subject: [PATCH 6/7] fixup
---
.../versions/0033_3_0_0_remove_redundant_index.py | 6 ------
1 file changed, 6 deletions(-)
diff --git a/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
index 5dab860dbaf8..fff72e4b0e9f 100644
--- a/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
+++ b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
@@ -39,17 +39,11 @@
def upgrade():
"""Apply Remove redundant index."""
- # ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("log", schema=None) as batch_op:
batch_op.drop_index("idx_log_dag")
- # ### end Alembic commands ###
-
def downgrade():
"""Unapply Remove redundant index."""
- # ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("log", schema=None) as batch_op:
batch_op.create_index("idx_log_dag", ["dag_id"], unique=False)
-
- # ### end Alembic commands ###
From 49dd65ea64d2fd471769e6d1b73824f31fea55dc Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 11:26:02 -0700
Subject: [PATCH 7/7] fixup! fixup
---
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/migrations-ref.rst | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256
index c48a05b94722..237c598ec1dc 100644
--- a/docs/apache-airflow/img/airflow_erd.sha256
+++ b/docs/apache-airflow/img/airflow_erd.sha256
@@ -1 +1 @@
-b937654c9ed767d8086d74e155d5bfc29ca6f44f8fcd8e3496fa603f37527cd9
\ No newline at end of file
+f4379048d3f13f35aaba824c00450c17ad4deea9af82b5498d755a12f8a85a37
\ No newline at end of file
diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst
index 3884db0564c3..ded2b290b5a4 100644
--- a/docs/apache-airflow/migrations-ref.rst
+++ b/docs/apache-airflow/migrations-ref.rst
@@ -39,7 +39,7 @@ Here's the list of all the Database Migrations that are executed via when you ru
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| Revision ID | Revises ID | Airflow Version | Description |
+=========================+==================+===================+==============================================================+
-| ``16cbcb1c8c36`` (head) | ``522625f6d606`` | ``3.0.0`` | Remove redundant index |
+| ``16cbcb1c8c36`` (head) | ``522625f6d606`` | ``3.0.0`` | Remove redundant index. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| ``522625f6d606`` | ``1cdc775ca98f`` | ``3.0.0`` | Add tables for backfill. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+