LT01: Fix alias alignment with templates and add alignment_coordinate_space config

.gitignore

@@ -3,6 +3,7 @@
 .idea
 /.sqlfluff
 **/.DS_Store
+.junie
 
 # Ignore Python cache and prebuilt things
 .cache

.pre-commit-config.yaml

@@ -70,7 +70,7 @@ repos:
     rev: 7.1.1
     hooks:
       - id: flake8
-        additional_dependencies: [flake8-black>=0.3.6]
+        additional_dependencies: [flake8-black>=0.3.7]
   - repo: https://github.com/pycqa/doc8
     rev: v1.1.2
     hooks:

CONTRIBUTING.md

@@ -337,13 +337,12 @@ need a [GitHub Personal Access Token](https://docs.github.com/en/authentication/
 scope, and read permissions on the "Metadata" scope. The reason we need
 both read & write access on the "Content" scope is that only tokens with
 write access can see _draft_ releases, which is what we need access to).
-All maintainers should have sufficient access to generate such a token:
+All maintainers should have sufficient access to generate such a token. Once
+generated, store it in your env (zshrc, etc) as `SQLFLUFF_GITHUB_TOKEN`. Then:
 
 ```shell
-source .venv/bin/activate
-export GITHUB_REPOSITORY_OWNER=sqlfluff
-export GITHUB_TOKEN=gho_xxxxxxxx # Change to your token with "repo" permissions.
-python util.py release 2.0.3 # Change to your release number
+make shell
+make release 3.4.2 # Change to your release number
 ```
 
 When all of the changes planned for the release have been merged, and

Makefile

@@ -1,7 +1,14 @@
-.PHONY: help build clean fresh shell start stop
+.PHONY: help build clean fresh release shell start stop
 
 .DEFAULT_GOAL := help
 
+# Set VERSION from first argument if provided
+VERSION := $(word 2,$(filter-out $(.DEFAULT_GOAL),$(MAKECMDGOALS)))
+
+# Treat flow name as a target (prevents make errors)
+$(VERSION):
+	@:
+
 help: ## Show this available targets
 	@grep -E '^[/a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
@@ -16,6 +23,9 @@ clean: ## Clean up all containers and images
 fresh: ## Build the development container from scratch
 	docker-compose build --no-cache development
 
+release: ## Release a new version
+	@python util.py release $(VERSION)
+
 shell: ## Start a bash session in the development container
 	docker-compose exec development bash
 

docker-compose.yml

@@ -4,8 +4,10 @@ services:
       context: .
       dockerfile: ./docker/development/Dockerfile
     environment:
-      - SSH_AUTH_SOCK=/ssh-agent
+      - GITHUB_REPOSITORY_OWNER=sqlfluff
+      - GITHUB_TOKEN=${SQLFLUFF_GITHUB_TOKEN}
       - POSTGRES_HOST=postgres
+      - SSH_AUTH_SOCK=/ssh-agent
     volumes:
       - .:/app
       - ./test/fixtures/dbt/profiles_yml:/root/.dbt

docs/source/configuration/layout.rst

@@ -167,11 +167,62 @@ the least obvious. The following example illustrates the impact it has.
    --    align_scope = file
    --    align_within = statement
 
+Templating and alignment coordinate space
+-----------------------------------------
+
+When using templating (e.g. Jinja), alignment is for human readability and
+stable diffs. SQLFluff aligns based on the source (visible) positions whenever
+templated (non-literal) segments are involved in the alignment scope. This
+prevents excessive padding caused by the rendered output being longer than the
+source template. If all segments are literal (non-templated), alignment uses
+the regular templated working positions.
+
+Example (Jinja templating, align alias expressions within a select clause):
+
+.. code-block:: jinja
+
+    select
+        {{ "longtemplated" }} as test_key,
+        b                     as b_col
+
+The alignment above is computed against the source text so that both lines line
+up visually in the editor, regardless of the rendered length of
+``{{ "longtemplated" }}``.
+
+Advanced: coordinate space override
+-----------------------------------
+
+You can optionally force the coordinate space either:
+
+1) via the alignment constraint suffix (available for `spacing_before` and
+   `spacing_after`), or
+2) via the `alignment_coordinate_space` key in layout config for the target type.
+
+.. code-block:: ini
+
+   [sqlfluff:layout:type:alias_expression]
+   spacing_before = align:alias_expression:select_clause:bracketed:source
+
+Alternatively, the equivalent can be configured more declaratively:
+
+.. code-block:: ini
+
+   [sqlfluff:layout:type:alias_expression]
+   spacing_before = align
+   align_within = select_clause
+   align_scope = bracketed
+   alignment_coordinate_space = source
+
+Supported values are ``source`` and ``templated``. In most cases, ``source`` is
+the recommended choice for readability.
+
+.. code-block:: sql
+
    WITH foo as (
       SELECT
          a,
          b,
-         c        AS first_column
+         c        AS first_column,
          d + e    AS second_column
    )
 
@@ -625,7 +676,7 @@ subsections of an :code:`ON` block with each other. If set to :code:`False`
 
 These can also be combined, so if :code:`indented_using_on` config is set to
 :code:`False`, :code:`indented_on_contents` is also set to :code:`False`, and
-:code:`allow_implicit_indents` is set tot :code:`True` then the SQL would
+:code:`allow_implicit_indents` is set to :code:`True` then the SQL would
 become:
 
 .. code-block:: sql

pyproject.toml

@@ -241,6 +241,7 @@ extend-select = ["I", "D"]
 # D418: Function/ Method decorated with @overload shouldn’t contain a docstring
 ignore = ["D107", "D105", "D418"]
 
+
 [tool.ruff.lint.isort]
 # Mark sqlfluff, test and it's plugins as known first party
 known-first-party = [

requirements_dev.txt

@@ -6,7 +6,7 @@ flake8
 flake8-docstrings
 pydocstyle!=6.2.0, !=6.2.1  # See: https://github.com/PyCQA/pydocstyle/issues/618
 black>=22.1.0
-flake8-black>=0.2.4
+flake8-black>=0.3.7
 ruff
 import-linter
 yamllint

src/sqlfluff/__main__.py

@@ -1,6 +1,6 @@
 """Export cli to __main__ for use like python -m sqlfluff."""
 
-from sqlfluff.cli.commands import cli
+from sqlfluff.cli.commands import cli  # pragma: no cover
 
-if __name__ == "__main__":
-    cli()
+if __name__ == "__main__":  # pragma: no cover
+    cli()  # pragma: no cover

src/sqlfluff/core/config/fluffconfig.py

@@ -120,8 +120,10 @@ def __init__(
         # If any existing configs are provided. Validate them:
         if configs:
             validate_config_dict(configs, "<provided configs>")
+        empty_config: ConfigMappingType = {"core": {}}
+        empty_overrides: ConfigMappingType = {}
         self._configs = nested_combine(
-            defaults, configs or {"core": {}}, overrides or {}
+            defaults, configs or empty_config, overrides or empty_overrides
         )
         # Some configs require special treatment
         self._configs["core"]["color"] = (

src/sqlfluff/core/helpers/string.py

@@ -7,7 +7,7 @@
 def curtail_string(s: str, length: int = 20) -> str:
     """Trim a string nicely to length."""
     if len(s) > length:
-        return s[:length] + "..."
+        return s[:length] + "..."  # pragma: no cover
     else:
         return s
 

src/sqlfluff/dialects/dialect_ansi.py

@@ -805,11 +805,16 @@ class FileSegment(BaseFileSegment):
     has no match_grammar.
     """
 
-    match_grammar = Delimited(
-        Ref("StatementSegment"),
-        delimiter=AnyNumberOf(Ref("DelimiterGrammar"), min_times=1),
-        allow_gaps=True,
-        allow_trailing=True,
+    # Allow leading & trailing delimiters plus runs of delimited statements.
+    match_grammar = Sequence(
+        AnyNumberOf(Ref("DelimiterGrammar")),
+        Delimited(
+            Ref("StatementSegment"),
+            delimiter=AnyNumberOf(Ref("DelimiterGrammar"), min_times=1),
+            allow_gaps=True,
+            allow_trailing=True,
+        ),
+        AnyNumberOf(Ref("DelimiterGrammar")),
     )
 
     def get_table_references(self) -> set[str]:

src/sqlfluff/dialects/dialect_bigquery.py

@@ -605,6 +605,7 @@ class FileSegment(BaseFileSegment):
     # NB: We don't need a match_grammar here because we're
     # going straight into instantiating it directly usually.
     match_grammar = Sequence(
+        AnyNumberOf(Ref("DelimiterGrammar")),
         Sequence(
             OneOf(
                 Ref("MultiStatementSegment"),
@@ -618,7 +619,7 @@ class FileSegment(BaseFileSegment):
                 Ref("StatementSegment"),
             ),
         ),
-        Ref("DelimiterGrammar", optional=True),
+        AnyNumberOf(Ref("DelimiterGrammar")),
     )
 
 
@@ -3128,10 +3129,24 @@ class CreateVectorIndexStatementSegment(BaseSegment):
                 Ref("IndexColumnDefinitionSegment"),
             ),
         ),
+        Ref("StoringSegment", optional=True),
         Ref("OptionsSegment"),
     )
 
 
+class StoringSegment(BaseSegment):
+    """The `STORING` clause for a `CREATE VECTOR INDEX` statement.
+
+    https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_vector_index_statement
+    """
+
+    type = "storing_segment"
+    match_grammar: Matchable = Sequence(
+        "STORING",
+        Bracketed(Delimited(Ref("SingleIdentifierGrammar"))),
+    )
+
+
 class DropVectorIndexStatementSegment(BaseSegment):
     """A `DROP VECTOR INDEX` statement.
 

src/sqlfluff/dialects/dialect_bigquery_keywords.py

@@ -278,6 +278,7 @@
 SOURCE
 STAGE
 START
+STORING
 STREAM
 STRICT
 SUNDAY

src/sqlfluff/dialects/dialect_clickhouse.py

@@ -1421,6 +1421,38 @@ class DropUserStatementSegment(ansi.DropUserStatementSegment):
     )
 
 
+class CreateUserStatementSegment(BaseSegment):
+    """A `CREATE USER` statement.
+
+    As specified in
+    https://clickhouse.com/docs/en/sql-reference/statements/create/user/
+    """
+
+    type = "create_user_statement"
+
+    match_grammar = Sequence(
+        "CREATE",
+        "USER",
+        Ref("IfNotExistsGrammar", optional=True),
+        Ref("SingleIdentifierGrammar"),
+        Ref("OnClusterClauseSegment", optional=True),
+        # IDENTIFIED BY 'password' or IDENTIFIED WITH ... BY ...
+        Sequence(
+            "IDENTIFIED",
+            Sequence(
+                "WITH",
+                Ref("SingleIdentifierGrammar"),
+                optional=True,
+            ),
+            Sequence(
+                "BY",
+                Ref("QuotedLiteralSegment"),
+            ),
+            optional=True,
+        ),
+    )
+
+
 class DropRoleStatementSegment(ansi.DropRoleStatementSegment):
     """A `DROP ROLE` statement.
 
@@ -2119,6 +2151,18 @@ class AlterTableStatementSegment(BaseSegment):
                     optional=True,
                 ),
             ),
+            # ALTER TABLE ... DROP PARTITION|PART partition_expr
+            Sequence(
+                "DROP", OneOf("PARTITION", "PART"), Ref("SingleIdentifierGrammar")
+            ),
+            # ALTER TABLE ... REPLACE PARTITION partition_expr FROM table1
+            Sequence(
+                "REPLACE",
+                "PARTITION",
+                Ref("SingleIdentifierGrammar"),
+                "FROM",
+                Ref("TableReferenceSegment"),
+            ),
         ),
     )
 
@@ -2128,6 +2172,7 @@ class StatementSegment(ansi.StatementSegment):
 
     match_grammar = ansi.StatementSegment.match_grammar.copy(
         insert=[
+            Ref("CreateUserStatementSegment"),
             Ref("CreateMaterializedViewStatementSegment"),
             Ref("DropDictionaryStatementSegment"),
             Ref("DropQuotaStatementSegment"),

src/sqlfluff/dialects/dialect_clickhouse_keywords.py

@@ -170,6 +170,7 @@
     "HIERARCHICAL",
     "HOUR",
     "ID",
+    "IDENTIFIED",
     "IF",
     "ILIKE",
     "IN",
@@ -219,6 +220,7 @@
     "OR",
     "OUTFILE",
     "OVERRIDE",
+    "PART",
     "PASTE",
     "POPULATE",
     "POSTGRESQL",

src/sqlfluff/dialects/dialect_databricks.py

@@ -1567,6 +1567,10 @@ class CommentOnStatementSegment(BaseSegment):
                 "VOLUME",
                 Ref("VolumeReferenceSegment"),
             ),
+            Sequence(
+                "COLUMN",
+                Ref("ColumnReferenceSegment"),
+            ),
             # TODO: Split out individual items if they have references
             Sequence(
                 OneOf(