Add date format filters to Jinja environment (#17451)

On its own this doesn't add all that much, but this is preparatory
work to be combined with the new data_interval_start template context
variables we are adding, without having to add the ds/ts/no-dash etc
permutations of all of them.

Author: ashb

airflow/macros/__init__.py

@@ -15,7 +15,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Macros."""
 import time  # noqa
 import uuid  # noqa
 from datetime import datetime, timedelta

airflow/templates.py

@@ -22,6 +22,11 @@
 class SandboxedEnvironment(jinja2.sandbox.SandboxedEnvironment):
     """SandboxedEnvironment for Airflow task templates."""
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.filters.update(FILTERS)
+
     def is_safe_attribute(self, obj, attr, value):
         """
         Allow access to ``_`` prefix vars (but not ``__``).
@@ -30,3 +35,32 @@ def is_safe_attribute(self, obj, attr, value):
         ``_``) whilst still blocking internal or truely private attributes (``__`` prefixed ones).
         """
         return not jinja2.sandbox.is_internal_attribute(obj, attr)
+
+
+def ds_filter(value):
+    return value.strftime('%Y-%m-%d')
+
+
+def ds_nodash_filter(value):
+    return value.strftime('%Y%m%d')
+
+
+def ts_filter(value):
+    return value.isoformat()
+
+
+def ts_nodash_filter(value):
+    return value.strftime('%Y%m%dT%H%M%S')
+
+
+def ts_nodash_with_tz_filter(value):
+    return value.isoformat().replace('-', '').replace(':', '')
+
+
+FILTERS = {
+    'ds': ds_filter,
+    'ds_nodash': ds_nodash_filter,
+    'ts': ts_filter,
+    'ts_nodash': ts_nodash_filter,
+    'ts_nodash_with_tz': ts_nodash_with_tz_filter,
+}

docs/apache-airflow/concepts/operators.rst

@@ -58,7 +58,7 @@ But there are many, many more - you can see the list of those in our :doc:`apach
 
 Jinja Templating
 ----------------
-Airflow leverages the power of `Jinja Templating <http://jinja.pocoo.org/docs/dev/>`_ and this can be a powerful tool to use in combination with :doc:`macros </macros-ref>`.
+Airflow leverages the power of `Jinja Templating <http://jinja.pocoo.org/docs/dev/>`_ and this can be a powerful tool to use in combination with :ref:`macros <templates-ref>`.
 
 For example, say you want to pass the execution date as an environment variable to a Bash script using the ``BashOperator``:
 
@@ -73,7 +73,7 @@ For example, say you want to pass the execution date as an environment variable
       env={"EXECUTION_DATE": date},
   )
 
-Here, ``{{ ds }}`` is a macro, and because the ``env`` parameter of the ``BashOperator`` is templated with Jinja, the execution date will be available as an environment variable named ``EXECUTION_DATE`` in your Bash script.
+Here, ``{{ ds }}`` is a templated variable, and because the ``env`` parameter of the ``BashOperator`` is templated with Jinja, the execution date will be available as an environment variable named ``EXECUTION_DATE`` in your Bash script.
 
 You can use Jinja templating with every parameter that is marked as "templated" in the documentation. Template substitution occurs just before the ``pre_execute`` function of your operator is called.
 

docs/apache-airflow/faq.rst

@@ -220,7 +220,7 @@ Airflow was developed as a solution for ETL needs. In the ETL world, you typical
 summarize data for 2016-02-19, You would do it at 2016-02-20 midnight UTC, which would be right after all data for
 2016-02-19 becomes available.
 
-This datetime value is available to you as :ref:`Macros<macros:default_variables>` as various forms in Jinja templated
+This datetime value is available to you as :ref:`Template variables<templates:variables>` with various formats in Jinja templated
 fields. They are also included in the context dictionary given to an Operator's execute function.
 
 .. code-block:: python

docs/apache-airflow/howto/operator/python.rst

@@ -47,7 +47,7 @@ Templating
 ^^^^^^^^^^
 
 Airflow passes in an additional set of keyword arguments: one for each of the
-:doc:`Jinja template variables <../../macros-ref>` and a ``templates_dict``
+:ref:`Jinja template variables <templates:variables>` and a ``templates_dict``
 argument.
 
 The ``templates_dict`` argument is templated, so each value in the dictionary

docs/apache-airflow/index.rst

@@ -110,7 +110,7 @@ unit of work and continuity.
 
     Operators and hooks <operators-and-hooks-ref>
     CLI <cli-and-env-variables-ref>
-    Macros <macros-ref>
+    Templates <templates-ref>
     Python API <python-api-ref>
     Stable REST API <stable-rest-api-ref>
     deprecated-rest-api-ref

docs/apache-airflow/redirects.txt

@@ -44,6 +44,7 @@ start.rst start/index.rst
 cli-ref.rst cli-and-env-variables-ref.rst
 _api/index.rst python-api-ref.rst
 rest-api-ref.rst deprecated-rest-api-ref.rst
+macros-ref.rst templates-ref.rst
 
 # Concepts
 concepts.rst concepts/index.rst

docs/apache-airflow/macros-ref.rst docs/apache-airflow/templates-ref.rst

@@ -15,20 +15,20 @@
     specific language governing permissions and limitations
     under the License.
 
-.. _macros:
+.. _templates-ref:
 
-Macros reference
-================
+Templates reference
+===================
 
-Variables and macros can be used in templates (see the :ref:`concepts:jinja-templating` section)
+Variables, macros and filters can be used in templates (see the :ref:`concepts:jinja-templating` section)
 
 The following come for free out of the box with Airflow.
 Additional custom macros can be added globally through :doc:`plugins`, or at a DAG level through the ``DAG.user_defined_macros`` argument.
 
-.. _macros:default_variables:
+.. _templates:variables:
 
-Default Variables
------------------
+Variables
+---------
 The Airflow engine passes a few variables by default that are accessible
 in all templates
 
@@ -105,14 +105,35 @@ For example, you could use expressions in your templates like ``{{ conn.my_conn_
 Just like with ``var`` it's possible to fetch a connection by string  (e.g. ``{{ conn.get('my_conn_id_'+index).host }}``
 ) or provide defaults (e.g ``{{ conn.get('my_conn_id', {"host": "host1", "login": "user1"}).host }}``)
 
+Filters
+-------
+
+Airflow defines the some Jinja filters that can be used to format values.
+
+For example, using ``{{ execution_date | ds }}`` will output the execution_date in the ``YYYY-MM-DD`` format.
+
+=====================  ============  ==================================================================
+Filter                 Operates on   Description
+=====================  ============  ==================================================================
+``ds``                 datetime      Format the datetime as ``YYYY-MM-DD``
+``ds_nodash``          datetime      Format the datetime as ``YYYYMMDD``
+``ts``                 datetime      Same as ``.isoformat()``, Example: ``2018-01-01T00:00:00+00:00``
+``ts_nodash``          datetime      Same as ``ts`` filter without ``-``, ``:`` or TimeZone info.
+                                     Example: ``20180101T000000``
+``ts_nodash_with_tz``  datetime      As ``ts`` filter without ``-`` or ``:``. Example
+                                     ``20180101T000000+0000``
+=====================  ============  ==================================================================
+
+
+.. _templates:macros:
+
 Macros
 ------
 Macros are a way to expose objects to your templates and live under the
 ``macros`` namespace in your templates.
 
 A few commonly used libraries and methods are made available.
 
-
 =================================   ==============================================
 Variable                            Description
 =================================   ==============================================
@@ -124,13 +145,12 @@ Variable                            Description
 ``macros.random``                   The standard lib's :mod:`random`
 =================================   ==============================================
 
-
 Some airflow specific macros are also defined:
 
 .. automodule:: airflow.macros
-    :show-inheritance:
     :members:
 
-.. autofunction:: airflow.macros.hive.closest_ds_partition
-.. autofunction:: airflow.macros.hive.max_partition
-.. _pendulum.Pendulum: https://pendulum.eustace.io/docs/1.x/#introduction
+.. automodule:: airflow.macros.hive
+    :members:
+
+.. _pendulum.Pendulum: https://pendulum.eustace.io/docs/2.x/#introduction

docs/apache-airflow/tutorial.rst

@@ -178,7 +178,7 @@ regarding custom filters have a look at the
 `Jinja Documentation <http://jinja.pocoo.org/docs/dev/api/#writing-filters>`_.
 
 For more information on the variables and macros that can be referenced
-in templates, make sure to read through the :doc:`macros-ref`.
+in templates, make sure to read through the :ref:`templates-ref`.
 
 Adding DAG and Tasks documentation
 ----------------------------------

tests/core/test_templates.py

@@ -17,6 +17,7 @@
 
 import jinja2
 import jinja2.exceptions
+import pendulum
 import pytest
 
 import airflow.templates
@@ -37,3 +38,19 @@ class Test:
 def test_private_access(env):
     with pytest.raises(jinja2.exceptions.SecurityError):
         env.from_string(r'{{ func.__code__ }}').render(func=test_private_access)
+
+
+@pytest.mark.parametrize(
+    ['name', 'expected'],
+    (
+        ('ds', '2012-07-24'),
+        ('ds_nodash', '20120724'),
+        ('ts', '2012-07-24T03:04:52+00:00'),
+        ('ts_nodash', '20120724T030452'),
+        ('ts_nodash_with_tz', '20120724T030452+0000'),
+    ),
+)
+def test_filters(env, name, expected):
+    when = pendulum.datetime(2012, 7, 24, 3, 4, 52, tz='UTC')
+    result = env.from_string('{{ date |' + name + ' }}').render(date=when)
+    assert result == expected