Line breaks not handled gracefully

This happened because the lexer got confused by the unknown Tag `imagekit` which comes from an addon. It was lexed and parsed as a variable and the syntax does not allow dynamic bindings for variables, only for Tags. This PR introduces an `T_UNKOWN_TAG` token, which is returned by the lexer when he sees an Identifier which is followed by one or more whitespaces and a colon.

The formatting issues was just a side effect of the wrong parsing.

Closes #120

src/main/java/de/arrobait/antlers/grammar/Antlers.bnf

@@ -117,6 +117,8 @@
         T_PERCENT="%"
 
         T_MODIFIER="regexp:add|add_query_param|add_slashes|alias|ampersand_list|antlers|ascii|as|at|background_position|backspace|bard_html|bard_items|bard_text|bool_string|camelize|cdata|ceil|chunk|collapse|collapse_whitespace|compact|console_log|contains|contains_all|contains_any|count|count_substring|dashify|days_ago|dd|ddd|debug|decode|deslugify|diff_for_humans|diff_for_owls|divide|dl|dump|embed_url|ends_with|ensure_left|ensure_right|entities|excerpt|explode|extension|favicon|first|flatten|flip|floor|format|format_localized|format_number|format_translated|full_urls|get|gravatar|group_by|has_lower_case|has_upper_case|hours_ago|image|in_array|insert|is_after|is_alpha|is_alphanumeric|is_array|is_before|is_between|is_blank|is_email|is_embeddable|is_empty|is_future|is_iterable|is_json|is_leap_year|is_lowercase|is_numberwang|is_numeric|iso_format|is_past|is_today|is_tomorrow|is_uppercase|is_url|is_weekday|is_weekend|is_yesterday|join|joinplode|kebab|key_by|last|lcfirst|length|limit|link|list|localize|lower|macro|mailto|mark|markdown|md5|merge|minutes_ago|mod|modify_date|months_ago|multiply|neatify|nl2br|obfuscate|obfuscate_email|offset|ol|option_list|output|pad|parse_url|partial|pathinfo|pluck|piped|plural|random|raw|rawurlencode|ray|read_time|regex_mark|regex_replace|relative|remove_left|remove_query_param|remove_right|repeat|replace|reverse|round|scope|safe_truncate|sanitize|seconds_ago|segment|sentence_list|set_query_parm|shrug|shuffle|singular|slugify|smartypants|snake|sort|spaceless|split|starts_with|strip_tags|str_pad|str_pad_both|str_pad_left|str_pad_right|studly|substr|subtract|sum|surround|swap_case|table|tidy|timestamp|timezone|title|to_bool|to_json|to_spaces|to_string|to_tabs|trackable_embed_url|trans|trans_choice|trim|truncate|type_of|ucfirst|ul|underscored|unique|upper|url|urldecode|urlencode|url_info|weeks_ago|where|widont|word_count|wrap|years_ago"
+
+        T_UNKNOWN_TAG='regexp:[:jletter:]*'
     ]
 
     extends("yaml_frontmatter")="de.arrobait.antlers.psi.AntlersFrontMatterMixin"
@@ -214,9 +216,9 @@ tag_node_close ::= node_opener (T_SLASH tag) node_closer { pin=2 }
 private tag_with_attributes ::= tag (tag_attribute_assignment | tag_taxonomy_condition)*
 
 tag ::= ['%'] (shorthand_tag | regular_tag)
-private shorthand_tag ::= regular_tag ':' tag_method_part { pin=2 }
+private shorthand_tag ::= regular_tag ':' tag_method_part { pin=3 }
 private regular_tag ::= tag_name
-tag_name ::= T_TAG
+tag_name ::= T_TAG | T_UNKNOWN_TAG
 tag_method_part ::= tag_method_name (':' tag_method_name)*
 private tag_method_name ::= T_TAG_METHOD_NAME
 

src/main/java/de/arrobait/antlers/grammar/AntlersLexer.flex

@@ -62,6 +62,7 @@ DOUBLE_QUOTED_STR_CONTENT = ([^\\\"]|\\([\\'\"/bfnrt]|u[a-fA-F0-9]{4}))+
 MODIFIERS=add|add_query_param|add_slashes|alias|ampersand_list|antlers|ascii|as|at|background_position|backspace|bard_html|bard_items|bard_text|bool_string|camelize|cdata|ceil|chunk|collapse|collapse_whitespace|compact|console_log|contains|contains_all|contains_any|count|count_substring|dashify|days_ago|dd|ddd|debug|decode|deslugify|diff_for_humans|diff_for_owls|divide|dl|dump|embed_url|ends_with|ensure_left|ensure_right|entities|excerpt|explode|extension|favicon|first|flatten|flip|floor|format|format_localized|format_number|format_translated|full_urls|get|gravatar|group_by|has_lower_case|has_upper_case|hours_ago|image|in_array|insert|is_after|is_alpha|is_alphanumeric|is_array|is_before|is_between|is_blank|is_email|is_embeddable|is_empty|is_future|is_iterable|is_json|is_leap_year|is_lowercase|is_numberwang|is_numeric|iso_format|is_past|is_today|is_tomorrow|is_uppercase|is_url|is_weekday|is_weekend|is_yesterday|join|joinplode|kebab|key_by|last|lcfirst|length|limit|link|list|localize|lower|macro|mailto|mark|markdown|md5|merge|minutes_ago|mod|modify_date|months_ago|multiply|neatify|nl2br|obfuscate|obfuscate_email|offset|ol|option_list|output|pad|parse_url|partial|pathinfo|pluck|piped|plural|random|raw|rawurlencode|ray|read_time|regex_mark|regex_replace|relative|remove_left|remove_query_param|remove_right|repeat|replace|reverse|round|scope|safe_truncate|sanitize|seconds_ago|segment|sentence_list|set_query_parm|shrug|shuffle|singular|slugify|smartypants|snake|sort|spaceless|split|starts_with|strip_tags|str_pad|str_pad_both|str_pad_left|str_pad_right|studly|substr|subtract|sum|surround|swap_case|table|tidy|timestamp|timezone|title|to_bool|to_json|to_spaces|to_string|to_tabs|trackable_embed_url|trans|trans_choice|trim|truncate|type_of|ucfirst|ul|underscored|unique|upper|url|urldecode|urlencode|url_info|weeks_ago|where|widont|word_count|wrap|years_ago
 
 TAG="%"?{TAG_NAMES}":"?
+UNKNOWN_TAG=[:jletter:]*
 TAG_NAMES=404|asset|assets|cache|can|collection|dd|dump|foreach|form|get_content|get_error|get_errors|get_files|glide|in|increment|installed|is|link|locales|loop|markdown|mix|mount_url|nav|nocache|not_found|oauth|obfuscate|parent|partial|protect|path|query|range|redirect|relate|rotate|route|scope|search|section|session|set|structure|svg|switch|taxonomy|theme|trans|trans_choice|user|users|vite|widont|yield|yields
 TAG_METHOD_NAME=[_A-Za-z][-_/0-9A-Za-z\.]*
 TAG_STRING_CONDITIONS=:contains|:doesnt_contain|:doesnt_end_with|:doesnt_exist|:doesnt_match|:doesnt_start_with|:ends_with|:equals|:exists|:gt|:gte|:in|:is|:is_after|:is_alpha|:is_alpha_numeric|:is_before|:is_email|:is_embeddable|:is_empty|:is_numberwang|:is_numeric|:is_url|:isnt|:isset|:lt|:lte|:matches|:not|:not_in|:null|:regex|:starts_with
@@ -261,9 +262,11 @@ FLOAT=[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+
     {RECURSIVE_CHILDREN} { yypushback(yylength()); pushState(RECURSIVE_CHILDREN); }
 
     {SLOT_COLON}         { yypushback(yylength()); pushState(PROPERTY_ACCESS); }
+    {IDENTIFIER}         { return T_IDENTIFIER; }
     {IDENTIFIER_DOT}     { yypushback(yylength()); pushState(PROPERTY_ACCESS); }
     {IDENTIFIER_COLON}   { yypushback(yylength()); pushState(PROPERTY_ACCESS); }
     {IDENTIFIER_BRACKET} { yypushback(yylength()); pushState(PROPERTY_ACCESS); }
+    {IDENTIFIER} / ({WHITE_SPACE}* ":") { yypushback(yylength()); pushState(TAG_EXPRESSION); }
 
     {INTEGER}            { return T_INTEGER; }
     {FLOAT}              { return T_FLOAT; }
@@ -321,11 +324,12 @@ FLOAT=[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+
 }
 
 <TAG_EXPRESSION> {
-    {WHITE_SPACE}  { pushState(TAG_EXPRESSION_ATTRIBUTE_LIST); return TokenType.WHITE_SPACE; }
-    "%"            { return T_PERCENT; }
-    ":"            { pushState(TAG_SHORTHAND); return T_COLON; }
-    {SLASH}        { return T_SLASH; }
-    {TAG_NAMES}    { return T_TAG; }
+    {WHITE_SPACE}*  { pushState(TAG_EXPRESSION_ATTRIBUTE_LIST); return TokenType.WHITE_SPACE; }
+    "%"             { return T_PERCENT; }
+    ":"             { pushState(TAG_SHORTHAND); return T_COLON; }
+    {SLASH}         { return T_SLASH; }
+    {TAG_NAMES}     { return T_TAG; }
+    {UNKNOWN_TAG}   { return T_UNKNOWN_TAG; }
 }
 
 <TAG_SHORTHAND> {TAG_METHOD_NAME} { return T_TAG_METHOD_NAME; }
@@ -338,7 +342,7 @@ FLOAT=[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+
     {SINGLE_QUOTE}  { pushState(SINGLE_STRING); return T_SINGLE_QUOTE; }
 }
 
-<ANTLERS_NODE, GROUP_BY, TAG_EXPRESSION_ATTRIBUTE_LIST, RECURSIVE_CHILDREN, PROPERTY_ACCESS, MODIFIER_LIST, TAG_EXPRESSION> {
+<GROUP_BY, RECURSIVE_CHILDREN, PROPERTY_ACCESS, MODIFIER_LIST> {
     {IDENTIFIER}  { return T_IDENTIFIER; }
 }
 
@@ -349,6 +353,7 @@ FLOAT=[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+
     "taxonomy:"              { return T_TAXONOMY; }
     "="                      { return T_OP_ASSIGN; }
     ":"                      { return T_DYNAMIC_BINDING; }
+    {IDENTIFIER}             { return T_IDENTIFIER; }
 }
 
 <RECURSIVE_CHILDREN> {

src/test/java/de/arrobait/antlers/lexer/TagsLexerTest.java

@@ -272,4 +272,39 @@ public void it_lexes_nav_multi_depth_collection() {
                 T_RD, "}}"
         );
     }
+
+    @Test
+    public void it_lexes_unknown_tag_from_addon() {
+        givenInput("{{ imagekit :src=\"foo\" }}");
+        thenTokensAre(
+                T_LD, "{{",
+                WHITE_SPACE, " ",
+                T_UNKNOWN_TAG, "imagekit",
+                WHITE_SPACE, " ",
+                T_DYNAMIC_BINDING, ":",
+                T_IDENTIFIER, "src",
+                T_OP_ASSIGN, "=",
+                T_DOUBLE_QUOTE, "\"",
+                T_STRING_CONTENT, "foo",
+                T_DOUBLE_QUOTE, "\"",
+                WHITE_SPACE, " ",
+                T_RD, "}}"
+        );
+
+        givenInput("{{ imagekit  :src=\"foo\" }}");
+        thenTokensAre(
+                T_LD, "{{",
+                WHITE_SPACE, " ",
+                T_UNKNOWN_TAG, "imagekit",
+                WHITE_SPACE, "  ",
+                T_DYNAMIC_BINDING, ":",
+                T_IDENTIFIER, "src",
+                T_OP_ASSIGN, "=",
+                T_DOUBLE_QUOTE, "\"",
+                T_STRING_CONTENT, "foo",
+                T_DOUBLE_QUOTE, "\"",
+                WHITE_SPACE, " ",
+                T_RD, "}}"
+        );
+    }
 }

src/test/java/de/arrobait/antlers/parser/AntlersParserSpecTest.java

@@ -70,6 +70,11 @@ public void testParseInterpolatedStatements() {
         doTest(true);
     }
 
+    @Test
+    public void testParseIssue120() {
+        doTest(true);
+    }
+
     @Test
     public void testParseLogicalExpressions() {
         doTest(true);

src/test/testData/parsing/ParseIssue120.antlers.html

@@ -0,0 +1,20 @@
+{{ nocache }} {{ collection from="training_log" status:is="published" limit="20"
+    sort="date:desc" }}
+<div class="flex {{ index === 0 ? 'col-span-3 row-span-3 sm:col-span-2 sm:row-span-2' : '' }}">
+  <a class="group relative flex" href="{{ url }}">
+    {{ if images }} {{ images limit="1" }} <img class="object-cover relative z-0" src="{{
+    imagekit :src=" path" n="home_thumbnail" }}" alt=""/> {{ /images }} {{ /if }} {{ if
+    index===0 }}
+    <div
+      class="pointer-events-none absolute bottom-0 right-0 z-20 bg-black bg-opacity-75 px-4 py-2 text-xs font-semibold text-cfhsv-blue group-hover:text-cfhsv-light-blue"
+    >
+      {{ title }}
+    </div>
+    {{ /if }}
+
+    <div
+      class="pointer-events-none absolute inset-0 z-10 bg-black opacity-20 transition-all duration-200 group-hover:opacity-0"
+    ></div>
+  </a>
+</div>
+{{ /collection }} {{ /nocache }}