WIP add heredoc templates; only EOF marker at the moment

README.md

@@ -6,7 +6,7 @@ tree-sitter grammar for the [HCL](https://github.com/hashicorp/hcl/blob/main/hcl
 
 Highlighting `example/example.hcl`:
 
-![Highlighting Example](https://i.imgur.com/sgFcxLK.png)
+![Highlighting Example](https://i.imgur.com/4XFVNVT.png)
 
 ## Developing
 
@@ -26,7 +26,7 @@ Given that some language features are still missing ( see TODO ) there are some
 nix-shell --run 'tree-sitter parse --quiet --stat example/real_world_stuff/*/*'
 ...
 ...
-Total parses: 1130; successful parses: 1053; failed parses: 77; success percentage: 93.19%
+Total parses: 1126; successful parses: 1110; failed parses: 16; success percentage: 98.58%
 
 ```
 
@@ -40,4 +40,6 @@ The aim is to build unit testcases from selected failure classes and slowly get
   * [x] add quoted templates
     * [x] add quoted template interpolations
     * [ ] add quoted template directives
-  * [ ] add heredoc templates
+  * [WIP] add heredoc templates
+    * support arbitary markers, at the moment for playground usage its only EOF
+

example/example.hcl

@@ -28,6 +28,11 @@ resource_1 "strlit1" "strlit2" {
   tpl1   = "prefix-${var.bar}"
   tpl2   = "prefix-${func("bar")}"
   tpl3   = "prefix-${func("nested-${var.bar}")}"
+  tpl4   = <<EOF
+    prefix
+    ${func("foo${ var.bar }")}
+    suffix
+  EOF
 
   nested_resource_1 {
     attr1 = 2

grammar.js

@@ -25,6 +25,7 @@ module.exports = grammar({
     $._template_literal_chunk,
     $._template_interpolation_start,
     $._template_interpolation_end,
+    $.heredoc_identifier,
   ],
 
   extras: $ => [
@@ -265,7 +266,7 @@ module.exports = grammar({
 
     template_expr: $ => choice(
       $.quoted_template,
-      // $.heredoc_template,
+      $.heredoc_template,
     ),
 
     quoted_template: $ => prec(PREC.quoted_template, seq(
@@ -278,6 +279,20 @@ module.exports = grammar({
       $._quoted_template_end,
     )),
 
+    // TODO user chosen identifiers
+    heredoc_template: $ => seq(
+      $.heredoc_start,
+      $.heredoc_identifier,
+      repeat(choice(
+        $.template_literal,
+        $.template_interpolation,
+        $.template_directive,
+      )),
+      $.heredoc_identifier,
+    ),
+
+    heredoc_start: $ => choice('<<', '<<-'),
+
     strip_marker: $ => '~',
 
     template_literal: $ => prec.right(repeat1(

src/grammar.json

@@ -1163,6 +1163,10 @@
         {
           "type": "SYMBOL",
           "name": "quoted_template"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "heredoc_template"
         }
       ]
     },
@@ -1203,6 +1207,56 @@
         ]
       }
     },
+    "heredoc_template": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "heredoc_start"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "heredoc_identifier"
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "template_literal"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "template_interpolation"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "template_directive"
+              }
+            ]
+          }
+        },
+        {
+          "type": "SYMBOL",
+          "name": "heredoc_identifier"
+        }
+      ]
+    },
+    "heredoc_start": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "<<"
+        },
+        {
+          "type": "STRING",
+          "value": "<<-"
+        }
+      ]
+    },
     "strip_marker": {
       "type": "STRING",
       "value": "~"
@@ -1354,6 +1408,10 @@
     {
       "type": "SYMBOL",
       "name": "_template_interpolation_end"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "heredoc_identifier"
     }
   ],
   "inline": [],

src/node-types.json

@@ -426,6 +426,42 @@
       ]
     }
   },
+  {
+    "type": "heredoc_start",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "heredoc_template",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "heredoc_identifier",
+          "named": true
+        },
+        {
+          "type": "heredoc_start",
+          "named": true
+        },
+        {
+          "type": "template_directive",
+          "named": true
+        },
+        {
+          "type": "template_interpolation",
+          "named": true
+        },
+        {
+          "type": "template_literal",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "index",
     "named": true,
@@ -616,6 +652,10 @@
       "multiple": false,
       "required": true,
       "types": [
+        {
+          "type": "heredoc_template",
+          "named": true
+        },
         {
           "type": "quoted_template",
           "named": true
@@ -792,6 +832,14 @@
     "type": "<",
     "named": false
   },
+  {
+    "type": "<<",
+    "named": false
+  },
+  {
+    "type": "<<-",
+    "named": false
+  },
   {
     "type": "<=",
     "named": false
@@ -848,6 +896,10 @@
     "type": "for",
     "named": false
   },
+  {
+    "type": "heredoc_identifier",
+    "named": true
+  },
   {
     "type": "identifier",
     "named": true

src/scanner.c

@@ -11,6 +11,7 @@ enum TokenType {
   TEMPLATE_LITERAL_CHUNK,
   TEMPLATE_INTERPOLATION_START,
   TEMPLATE_INTERPOLATION_END,
+  HEREDOC_IDENTIFIER,
 };
 
 static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
@@ -202,8 +203,24 @@ bool scanner_scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
     }
   }
 
+  // handle heredoc identifier
+  if (valid_symbols[HEREDOC_IDENTIFIER]) {
+    if (lexer->lookahead != 'E') {
+       if (valid_symbols[TEMPLATE_LITERAL_CHUNK]) return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+    }
+    advance(lexer);
+    if (lexer->lookahead != 'O') {
+       if (valid_symbols[TEMPLATE_LITERAL_CHUNK]) return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+    }
+    advance(lexer);
+    if (lexer->lookahead != 'F') {
+       if (valid_symbols[TEMPLATE_LITERAL_CHUNK]) return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+    }
+    return accept_and_advance(lexer, HEREDOC_IDENTIFIER);
+  }
+
   // handle all other quoted template or string literal characters
-  if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && scanner->in_quoted_context) {
+  if (valid_symbols[TEMPLATE_LITERAL_CHUNK]) {
     return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
   }