fmt tests; properly define string literals; work on scanner

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*.{c,txt,js}]
+indent_style = space
+indent_size = 2
+tab_width = 8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8

README.md

@@ -6,7 +6,7 @@ tree-sitter grammar for the [HCL](https://github.com/hashicorp/hcl/blob/main/hcl
 
 Highlighting `example/example.hcl`:
 
-![Highlighting Example](https://i.imgur.com/M1PZkqL.png)
+![Highlighting Example](https://i.imgur.com/sgFcxLK.png)
 
 ## Developing
 
@@ -16,17 +16,6 @@ It is recommended to use `nix` to fulfill all development dependencies. To activ
 
 To run tests simply run `nix-shell --run 'tree-sitter test'`.
 
-## Quoted Template Expressions
-
-In principle it is allowed to contain arbitary expressions in quoted template interpolations. Consider for example:
-
-```hcl
-foo = "prefix-${func(\"bar\")}"
-```
-
-To make parsing a little easier, this parser only checks for valid escape sequences and template chars.
-When using this parser one would have to take the content of a template interpolation, unescape it and parse it again to get the syntax tree. The same applies to template directives.
-
 ## Todo
 
 * [ ] use [Unicode® Standard Annex #31](https://www.unicode.org/reports/tr31/) (augmented with '-') for identifiers

example/example.hcl

@@ -26,7 +26,8 @@ resource_1 "strlit1" "strlit2" {
   esc1   = "\" \t \UFF11FF22 \uFFFF \n"
   esc2   = "$${} %%{}"
   tpl1   = "prefix-${var.bar}"
-  tpl2   = "prefix-${func(\"bar\")}"
+  tpl2   = "prefix-${func("bar")}"
+  tpl3   = "prefix-${func("nested-${var.bar}")}"
 
   nested_resource_1 {
     attr1 = 2

grammar.js

@@ -25,9 +25,11 @@ module.exports = grammar({
   ],
 
   externals: $ => [
-    $._template_char,
+    $._quoted_template_start,
-    $._template_char_in_interpolation,
+    $._quoted_template_end,
-    $.escape_sequence,
+    $._template_literal_chunk,
+    $._template_interpolation_start,
+    $._template_interpolation_end,
   ],
 
   extras: $ => [
@@ -54,11 +56,14 @@ module.exports = grammar({
     block: $ => seq(
       field('name', $.identifier),
       repeat(choice($.string_lit, $.identifier)),
-      '{',
+      $._block_start,
       optional($.body),
-      '}',
+      $._block_end,
     ),
 
+    _block_start: $ => '{',
+    _block_end: $ => '}',
+
     // TODO: not to spec but good enough for now
     identifier: $ => token(seq(
       unicodeLetter,
@@ -66,21 +71,21 @@ module.exports = grammar({
     )),
 
     expression: $ => choice(
-      $.expr_term,
+      $._expr_term,
       $.operation,
       $.conditional,
     ),
 
-    expr_term: $ => choice(
+    _expr_term: $ => choice(
       $.literal_value,
       $.template_expr,
       $.collection_value,
       $.variable_expr,
       $.function_call,
       $.for_expr,
-      seq($.expr_term, $.index),
+      seq($._expr_term, $.index),
-      seq($.expr_term, $.get_attr),
+      seq($._expr_term, $.get_attr),
-      seq($.expr_term, $.splat),
+      seq($._expr_term, $.splat),
       seq('(', $.expression, ')'),
     ),
 
@@ -88,6 +93,7 @@ module.exports = grammar({
       $.numeric_lit,
       $.bool_lit,
       $.null_lit,
+      $.string_lit,
     ),
 
     numeric_lit: $ => /[0-9]+(\.[0-9]+([eE][-+]?[0-9]+)?)?/,
@@ -96,7 +102,12 @@ module.exports = grammar({
 
     null_lit: $ => 'null',
 
-    // string_lit is defined as quoted template
+    string_lit: $ => seq(
+      $._quoted_template_start,
+      $.template_literal,
+      $._quoted_template_end,
+    ),
+
 
     collection_value: $ => choice(
       $.tuple,
@@ -113,14 +124,17 @@ module.exports = grammar({
     ),
 
     object: $ => seq(
-      '{',
+      $._object_start,
       optional(seq(
         $.object_elem,
         repeat(seq(',', $.object_elem)),
       )),
-      '}',
+      $._object_end,
     ),
 
+    _object_start: $ => '{',
+    _object_end: $ => '}',
+
     object_elem: $ => seq(
       choice($.identifier, $.expression),
       choice('=', ':'),
@@ -157,14 +171,14 @@ module.exports = grammar({
     ),
 
     for_object_expr: $ => seq(
-      '{',
+      $._object_start,
       $.for_intro,
       $.expression,
       '=>',
       $.expression,
       optional($.ellipsis),
       optional($.for_cond),
-      '}',
+      $._object_end,
     ),
 
     for_intro: $ => seq(
@@ -185,11 +199,14 @@ module.exports = grammar({
 
     function_call: $ => seq(
       field('name', $.identifier),
-      '(',
+      $._function_call_start,
       optional($.function_arguments),
-      ')'
+      $._function_call_end,
     ),
 
+    _function_call_start: $ => '(',
+    _function_call_end: $ => ')',
+
     function_arguments: $ => seq(
       $.expression,
       repeat(seq(',', $.expression)),
@@ -208,7 +225,7 @@ module.exports = grammar({
 
     operation: $ => choice($.unary_operation, $.binary_operation),
 
-    unary_operation: $ => prec.left(PREC.unary, seq(choice('-', '!'), $.expr_term)),
+    unary_operation: $ => prec.left(PREC.unary, seq(choice('-', '!'), $._expr_term)),
 
     binary_operation: $ => {
       const table = [
@@ -221,7 +238,7 @@ module.exports = grammar({
       ];
 
       return choice(...table.map(([precedence, operator]) =>
-        prec.left(precedence, seq($.expr_term, operator, $.expr_term),
+        prec.left(precedence, seq($._expr_term, operator, $._expr_term),
         ))
       );
     },
@@ -231,31 +248,37 @@ module.exports = grammar({
       // $.heredoc_template,
     ),
 
-    string_lit: $ => seq(
+    quoted_template: $ => seq(
-      '"', 
+      $._quoted_template_start,
       repeat(choice(
-        $._template_char, 
+        $.template_literal,
-        $.escape_sequence,
+        $.template_interpolation,
+        $.template_directive,
       )),
-      '"',
+      $._quoted_template_end,
     ),
 
-    quoted_template: $ => seq(
+    strip_marker: $ => '~',
-      '"', 
+
-      repeat(choice(
+    template_literal: $ => prec.right(repeat1(
-        $._template_char, 
+      $._template_literal_chunk,
-        $.escape_sequence,
+    )),
-        $.template_interpolation,
-      )), 
-      '"',
-    ),
 
     template_interpolation: $ => seq(
-      choice('${', '${~'),
+      $._template_interpolation_start,
-      repeat(choice($._template_char_in_interpolation, $.escape_sequence)),
+      optional($.strip_marker),
-      choice('}', '~}'),
+      $.expression,
+      optional($.strip_marker),
+      $._template_interpolation_end,
     ),
 
+    // TODO
+    template_directive: $ => choice(
+      //$.template_for,
+      //$.template_if,
+    ),
+
+
     // http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890
     comment: $ => token(choice(
       seq('#', /.*/),

shell.nix

@@ -13,7 +13,7 @@ in
     ];
     shellHook = ''
 PATH=./node_modules/.bin:$PATH
-command -v tree-sitter >/dev/null 2>&1 || npm install tree-sitter-cli
+command -v tree-sitter >/dev/null 2>&1 || npm install tree-sitter-cli@0.19.5
     '';
   }
 

src/grammar.json

@@ -75,8 +75,8 @@
           }
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "{"
+          "name": "_block_start"
         },
         {
           "type": "CHOICE",
@@ -91,11 +91,19 @@
           ]
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "}"
+          "name": "_block_end"
         }
       ]
     },
+    "_block_start": {
+      "type": "STRING",
+      "value": "{"
+    },
+    "_block_end": {
+      "type": "STRING",
+      "value": "}"
+    },
     "identifier": {
       "type": "TOKEN",
       "content": {
@@ -133,7 +141,7 @@
       "members": [
         {
           "type": "SYMBOL",
-          "name": "expr_term"
+          "name": "_expr_term"
         },
         {
           "type": "SYMBOL",
@@ -145,7 +153,7 @@
         }
       ]
     },
-    "expr_term": {
+    "_expr_term": {
       "type": "CHOICE",
       "members": [
         {
@@ -177,7 +185,7 @@
           "members": [
             {
               "type": "SYMBOL",
-              "name": "expr_term"
+              "name": "_expr_term"
             },
             {
               "type": "SYMBOL",
@@ -190,7 +198,7 @@
           "members": [
             {
               "type": "SYMBOL",
-              "name": "expr_term"
+              "name": "_expr_term"
             },
             {
               "type": "SYMBOL",
@@ -203,7 +211,7 @@
           "members": [
             {
               "type": "SYMBOL",
-              "name": "expr_term"
+              "name": "_expr_term"
             },
             {
               "type": "SYMBOL",
@@ -244,6 +252,10 @@
         {
           "type": "SYMBOL",
           "name": "null_lit"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "string_lit"
         }
       ]
     },
@@ -268,6 +280,23 @@
       "type": "STRING",
       "value": "null"
     },
+    "string_lit": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "_quoted_template_start"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "template_literal"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "_quoted_template_end"
+        }
+      ]
+    },
     "collection_value": {
       "type": "CHOICE",
       "members": [
@@ -331,8 +360,8 @@
       "type": "SEQ",
       "members": [
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "{"
+          "name": "_object_start"
         },
         {
           "type": "CHOICE",
@@ -368,11 +397,19 @@
           ]
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "}"
+          "name": "_object_end"
         }
       ]
     },
+    "_object_start": {
+      "type": "STRING",
+      "value": "{"
+    },
+    "_object_end": {
+      "type": "STRING",
+      "value": "}"
+    },
     "object_elem": {
       "type": "SEQ",
       "members": [
@@ -554,8 +591,8 @@
       "type": "SEQ",
       "members": [
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "{"
+          "name": "_object_start"
         },
         {
           "type": "SYMBOL",
@@ -598,8 +635,8 @@
           ]
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "}"
+          "name": "_object_end"
         }
       ]
     },
@@ -678,8 +715,8 @@
           }
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "("
+          "name": "_function_call_start"
         },
         {
           "type": "CHOICE",
@@ -694,11 +731,19 @@
           ]
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": ")"
+          "name": "_function_call_end"
         }
       ]
     },
+    "_function_call_start": {
+      "type": "STRING",
+      "value": "("
+    },
+    "_function_call_end": {
+      "type": "STRING",
+      "value": ")"
+    },
     "function_arguments": {
       "type": "SEQ",
       "members": [
@@ -811,7 +856,7 @@
           },
           {
             "type": "SYMBOL",
-            "name": "expr_term"
+            "name": "_expr_term"
           }
         ]
       }
@@ -827,7 +872,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               },
               {
                 "type": "CHOICE",
@@ -848,7 +893,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               }
             ]
           }
@@ -861,7 +906,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               },
               {
                 "type": "CHOICE",
@@ -878,7 +923,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               }
             ]
           }
@@ -891,7 +936,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               },
               {
                 "type": "CHOICE",
@@ -916,7 +961,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               }
             ]
           }
@@ -929,7 +974,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               },
               {
                 "type": "CHOICE",
@@ -946,7 +991,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               }
             ]
           }
@@ -959,7 +1004,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               },
               {
                 "type": "CHOICE",
@@ -972,7 +1017,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               }
             ]
           }
@@ -985,7 +1030,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               },
               {
                 "type": "CHOICE",
@@ -998,7 +1043,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "expr_term"
+                "name": "_expr_term"
               }
             ]
           }
@@ -1014,41 +1059,12 @@
         }
       ]
     },
-    "string_lit": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "\""
-        },
-        {
-          "type": "REPEAT",
-          "content": {
-            "type": "CHOICE",
-            "members": [
-              {
-                "type": "SYMBOL",
-                "name": "_template_char"
-              },
-              {
-                "type": "SYMBOL",
-                "name": "escape_sequence"
-              }
-            ]
-          }
-        },
-        {
-          "type": "STRING",
-          "value": "\""
-        }
-      ]
-    },
     "quoted_template": {
       "type": "SEQ",
       "members": [
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "\""
+          "name": "_quoted_template_start"
         },
         {
           "type": "REPEAT",
@@ -1057,72 +1073,85 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "_template_char"
+                "name": "template_literal"
-              },
-              {
-                "type": "SYMBOL",
-                "name": "escape_sequence"
               },
               {
                 "type": "SYMBOL",
                 "name": "template_interpolation"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "template_directive"
               }
             ]
           }
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "\""
+          "name": "_quoted_template_end"
         }
       ]
     },
+    "strip_marker": {
+      "type": "STRING",
+      "value": "~"
+    },
+    "template_literal": {
+      "type": "PREC_RIGHT",
+      "value": 0,
+      "content": {
+        "type": "REPEAT1",
+        "content": {
+          "type": "SYMBOL",
+          "name": "_template_literal_chunk"
+        }
+      }
+    },
     "template_interpolation": {
       "type": "SEQ",
       "members": [
         {
-          "type": "CHOICE",
+          "type": "SYMBOL",
-          "members": [
+          "name": "_template_interpolation_start"
-            {
-              "type": "STRING",
-              "value": "${"
-            },
-            {
-              "type": "STRING",
-              "value": "${~"
-            }
-          ]
-        },
-        {
-          "type": "REPEAT",
-          "content": {
-            "type": "CHOICE",
-            "members": [
-              {
-                "type": "SYMBOL",
-                "name": "_template_char_in_interpolation"
-              },
-              {
-                "type": "SYMBOL",
-                "name": "escape_sequence"
-              }
-            ]
-          }
         },
         {
           "type": "CHOICE",
           "members": [
             {
-              "type": "STRING",
+              "type": "SYMBOL",
-              "value": "}"
+              "name": "strip_marker"
             },
             {
-              "type": "STRING",
+              "type": "BLANK"
-              "value": "~}"
             }
           ]
+        },
+        {
+          "type": "SYMBOL",
+          "name": "expression"
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "strip_marker"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "SYMBOL",
+          "name": "_template_interpolation_end"
         }
       ]
     },
+    "template_directive": {
+      "type": "CHOICE",
+      "members": []
+    },
     "comment": {
       "type": "TOKEN",
       "content": {
@@ -1211,15 +1240,23 @@
   "externals": [
     {
       "type": "SYMBOL",
-      "name": "_template_char"
+      "name": "_quoted_template_start"
     },
     {
       "type": "SYMBOL",
-      "name": "_template_char_in_interpolation"
+      "name": "_quoted_template_end"
     },
     {
       "type": "SYMBOL",
-      "name": "escape_sequence"
+      "name": "_template_literal_chunk"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_template_interpolation_start"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_template_interpolation_end"
     }
   ],
   "inline": [],

src/node-types.json

@@ -49,7 +49,43 @@
       "required": true,
       "types": [
         {
-          "type": "expr_term",
+          "type": "collection_value",
+          "named": true
+        },
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "for_expr",
+          "named": true
+        },
+        {
+          "type": "function_call",
+          "named": true
+        },
+        {
+          "type": "get_attr",
+          "named": true
+        },
+        {
+          "type": "index",
+          "named": true
+        },
+        {
+          "type": "literal_value",
+          "named": true
+        },
+        {
+          "type": "splat",
+          "named": true
+        },
+        {
+          "type": "template_expr",
+          "named": true
+        },
+        {
+          "type": "variable_expr",
           "named": true
         }
       ]
@@ -163,7 +199,7 @@
     }
   },
   {
-    "type": "expr_term",
+    "type": "expression",
     "named": true,
     "fields": {},
     "children": {
@@ -175,7 +211,7 @@
           "named": true
         },
         {
-          "type": "expr_term",
+          "type": "conditional",
           "named": true
         },
         {
@@ -202,6 +238,10 @@
           "type": "literal_value",
           "named": true
         },
+        {
+          "type": "operation",
+          "named": true
+        },
         {
           "type": "splat",
           "named": true
@@ -217,29 +257,6 @@
       ]
     }
   },
-  {
-    "type": "expression",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": false,
-      "required": true,
-      "types": [
-        {
-          "type": "conditional",
-          "named": true
-        },
-        {
-          "type": "expr_term",
-          "named": true
-        },
-        {
-          "type": "operation",
-          "named": true
-        }
-      ]
-    }
-  },
   {
     "type": "for_cond",
     "named": true,
@@ -456,6 +473,10 @@
         {
           "type": "numeric_lit",
           "named": true
+        },
+        {
+          "type": "string_lit",
+          "named": true
         }
       ]
     }
@@ -522,12 +543,16 @@
       "required": false,
       "types": [
         {
-          "type": "escape_sequence",
+          "type": "template_directive",
           "named": true
         },
         {
           "type": "template_interpolation",
           "named": true
+        },
+        {
+          "type": "template_literal",
+          "named": true
         }
       ]
     }
@@ -556,16 +581,21 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
-          "type": "escape_sequence",
+          "type": "template_literal",
           "named": true
         }
       ]
     }
   },
+  {
+    "type": "template_directive",
+    "named": true,
+    "fields": {}
+  },
   {
     "type": "template_expr",
     "named": true,
@@ -587,15 +617,24 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
-          "type": "escape_sequence",
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "strip_marker",
           "named": true
         }
       ]
     }
   },
+  {
+    "type": "template_literal",
+    "named": true,
+    "fields": {}
+  },
   {
     "type": "tuple",
     "named": true,
@@ -616,11 +655,47 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": false,
+      "multiple": true,
       "required": true,
       "types": [
         {
-          "type": "expr_term",
+          "type": "collection_value",
+          "named": true
+        },
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "for_expr",
+          "named": true
+        },
+        {
+          "type": "function_call",
+          "named": true
+        },
+        {
+          "type": "get_attr",
+          "named": true
+        },
+        {
+          "type": "index",
+          "named": true
+        },
+        {
+          "type": "literal_value",
+          "named": true
+        },
+        {
+          "type": "splat",
+          "named": true
+        },
+        {
+          "type": "template_expr",
+          "named": true
+        },
+        {
+          "type": "variable_expr",
           "named": true
         }
       ]
@@ -649,18 +724,6 @@
     "type": "!=",
     "named": false
   },
-  {
-    "type": "\"",
-    "named": false
-  },
-  {
-    "type": "${",
-    "named": false
-  },
-  {
-    "type": "${~",
-    "named": false
-  },
   {
     "type": "%",
     "named": false
@@ -753,10 +816,6 @@
     "type": "ellipsis",
     "named": true
   },
-  {
-    "type": "escape_sequence",
-    "named": true
-  },
   {
     "type": "false",
     "named": false
@@ -785,6 +844,10 @@
     "type": "numeric_lit",
     "named": true
   },
+  {
+    "type": "strip_marker",
+    "named": true
+  },
   {
     "type": "true",
     "named": false
@@ -800,9 +863,5 @@
   {
     "type": "}",
     "named": false
-  },
-  {
-    "type": "~}",
-    "named": false
   }
 ]

src/scanner.c

@@ -1,144 +1,204 @@
 #include <tree_sitter/parser.h>
+
 #include <wctype.h>
+#include <string.h>
+#include <stdio.h>
 
 enum TokenType {
-  TEMPLATE_CHAR,
+  QUOTED_TEMPLATE_START,
-  TEMPLATE_CHAR_IN_INTERPOLATION,
+  QUOTED_TEMPLATE_END,
-  ESCAPE_SEQUENCE
+  TEMPLATE_LITERAL_CHUNK,
+  TEMPLATE_INTERPOLATION_START,
+  TEMPLATE_INTERPOLATION_END,
 };
 
 static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
 
-static bool accept_template_char_inplace(TSLexer *lexer) {
+static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
-    lexer->result_symbol = TEMPLATE_CHAR;
+
-    return true;
+static bool accept_inplace(TSLexer *lexer, enum TokenType token) {
+  lexer->result_symbol = token;
+  return true;
 }
 
-static bool advance_and_accept_template_char(TSLexer *lexer) {
+static bool accept_and_advance(TSLexer *lexer, enum TokenType token) {
-    advance(lexer);
+  advance(lexer);
-    return accept_template_char_inplace(lexer);
+  return accept_inplace(lexer, token);
-}
-
-static bool advance_and_accept_escape_sequence(TSLexer *lexer) {
-    advance(lexer);
-    lexer->result_symbol = ESCAPE_SEQUENCE;
-    return true;
-}
-
-static bool advance_and_accept_template_char_in_interpolation(TSLexer *lexer) {
-    advance(lexer);
-    lexer->result_symbol = TEMPLATE_CHAR_IN_INTERPOLATION;
-    return true;
 }
 
 static bool consume_wxdigit(TSLexer *lexer) {
-    advance(lexer);
+  advance(lexer);
-    return iswxdigit(lexer->lookahead);
+  return iswxdigit(lexer->lookahead);
 }
 
-// scan escape sequences \n \t \r \\ \" \uHHHH or \UHHHHHHHHA where H is a hex digit
+typedef struct Scanner {
-// assumes that the leading character is '\'
+  bool in_template_interpolation;
-static bool scan_backslash_escape_sequence(TSLexer *lexer) {
+  bool in_quoted_context;
-  advance(lexer);
+  int template_interpolation_depth;
-  switch (lexer->lookahead) {
+  int quoted_context_depth;
-    case '"':
+} Scanner;
-    case 'n':
+
-    case 'r':
+void print_debug_info(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
-    case 't':
+  printf("\nDEBUG INFO START\n\n");
-    case '\\':
+  printf("currently at: '%c'\n\n", lexer->lookahead);
-      return advance_and_accept_escape_sequence(lexer);
+  printf("could be one of\n");
-    case 'u':
+  printf("quoted_template_start: %x\n", valid_symbols[QUOTED_TEMPLATE_START]);
-      for (int i = 0; i < 4; i++) {
+  printf("quoted_template_end: %x\n", valid_symbols[QUOTED_TEMPLATE_END]);
-        if (!consume_wxdigit(lexer)) return false;
+  printf("template_literal_chunk: %x", valid_symbols[TEMPLATE_LITERAL_CHUNK]);
-      }
+  printf("template_interpolation_start: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_START]);
-      return advance_and_accept_escape_sequence(lexer);
+  printf("template_interpolation_end: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_END]);
-    case 'U':
+  printf("\n");
-      for (int i = 0; i < 8; i++) {
+  printf("scanner state:\n");
-        if (!consume_wxdigit(lexer)) return false;
+  printf("in_template_interpolation %x\n", scanner->in_template_interpolation);
-      }
+  printf("in_quoted_context %x\n", scanner->in_quoted_context);
-      return advance_and_accept_escape_sequence(lexer);
+  printf("template_interpolation_depth %x\n", scanner->template_interpolation_depth);
-    default:
+  printf("quoted_context_depth %x\n", scanner->quoted_context_depth);
-      return false;
+  printf("\nDEBUG INFO END\n\n");
+}
+
+// some helpers to keep track of template depth
+void scanner_enter_interpolation_context(Scanner *scanner) {
+  scanner->template_interpolation_depth++;
+  scanner->in_template_interpolation = true;
+  scanner->in_quoted_context = false;
+}
+
+void scanner_exit_interpolation_context(Scanner *scanner) {
+  scanner->template_interpolation_depth--;
+  scanner->in_template_interpolation = false;
+  scanner->in_quoted_context = true;
+}
+
+void scanner_enter_quoted_context(Scanner *scanner) {
+  scanner->quoted_context_depth++;
+  scanner->in_quoted_context = true;
+  scanner->in_template_interpolation = false;
+}
+
+void scanner_exit_quoted_context(Scanner *scanner) {
+  scanner->quoted_context_depth--;
+  scanner->in_quoted_context = false;
+  // check if we are reentering a nested template interpolation
+  if (scanner->template_interpolation_depth > 0) {
+    scanner->in_template_interpolation = true;
   }
 }
 
-// may accept multiple characters like %% as a 'template literal chunk'
+bool must_escape_in_nested_quoted_context(char c) {
-// assumes that the leading character is '%' or '$'
+  switch (c) {
-static bool scan_template_literal_chunk_or_template_escape_sequence(TSLexer *lexer) {
-  const leading_char = lexer->lookahead;
-
-  advance(lexer);
-  // reject %{ because its the start of template directives
-  if (lexer->lookahead == '{') return false;
-  if (lexer->lookahead == leading_char) {
-    advance(lexer);
-    // accept %%{ as escape sequence
-    if (lexer->lookahead == '{') return advance_and_accept_escape_sequence(lexer);
-  }
-  // accept % and %% as template chars
-  return accept_template_char_inplace(lexer);
-}
-
-static bool scan_template_char_or_escape_sequence(TSLexer *lexer) {
-  switch (lexer->lookahead) {
-    case '"':
     case '\n':
     case '\r':
     case '\t':
-      return false;
+      return true;
-    case '\\':
-      return scan_backslash_escape_sequence(lexer);
-    case '$':
-    case '%':
-      return scan_template_literal_chunk_or_template_escape_sequence(lexer);
     default:
-      return advance_and_accept_template_char(lexer);
+      return false;
   }
 }
 
-static bool scan_template_char_in_interpolation_or_escape_sequence(TSLexer *lexer) {
+bool scanner_scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
-  switch (lexer->lookahead) {
+  // literal newlines are not allowed inside a quoted context
-    case '"':
+  if (must_escape_in_nested_quoted_context(lexer->lookahead) && scanner->quoted_context_depth > 0) {
-    case '\n':
+    return false;
-    case '\r':
-    case '\t':
-    // no template interpolation chars are allowed in template interpolations ( even escaped )
-    case '$':
-    case '%':
-    case '~':
-      return false;
-    // '}' ends the template interpolation
-    case '}':
-      return false;
-    case '\\':
-      return scan_backslash_escape_sequence(lexer);
-    default:
-      return advance_and_accept_template_char_in_interpolation(lexer);
   }
-}
+  while (iswspace(lexer->lookahead) && !scanner->in_quoted_context) skip(lexer);
 
-// scanner
+  // manage quoted context
-
+  if (valid_symbols[QUOTED_TEMPLATE_START] && lexer->lookahead == '"') {
-void *tree_sitter_hcl_external_scanner_create() { return NULL; }
+    scanner_enter_quoted_context(scanner);
-void tree_sitter_hcl_external_scanner_destroy(void *p) {}
+    return accept_and_advance(lexer, QUOTED_TEMPLATE_START);
-void tree_sitter_hcl_external_scanner_reset(void *p) {}
-unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) { return 0; }
-void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
-
-
-bool tree_sitter_hcl_external_scanner_scan(
-  void *p,
-  TSLexer *lexer,
-  const bool *valid_symbols
-) {
-  // when scanning string literals or quoted template literals that are not in an template directive
-  if (valid_symbols[TEMPLATE_CHAR] && valid_symbols[ESCAPE_SEQUENCE]) {
-    return scan_template_char_or_escape_sequence(lexer);
   }
-  // quoted template literals currently inside a template directive
+  if (valid_symbols[QUOTED_TEMPLATE_END] && lexer->lookahead == '"') {
-  if (valid_symbols[TEMPLATE_CHAR_IN_INTERPOLATION] && valid_symbols[ESCAPE_SEQUENCE]) {
+    scanner_exit_quoted_context(scanner);
-    return scan_template_char_in_interpolation_or_escape_sequence(lexer);
+    return accept_and_advance(lexer, QUOTED_TEMPLATE_END);
+  }
+
+  if (valid_symbols[TEMPLATE_INTERPOLATION_START] && lexer->lookahead == '$') {
+    advance(lexer);
+    if (lexer->lookahead == '{') {
+      scanner_enter_interpolation_context(scanner);
+      return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_START);
+    }
+    if (valid_symbols[TEMPLATE_LITERAL_CHUNK]) {
+      // try to scan escape sequence
+      if (lexer->lookahead == '$') {
+        advance(lexer);
+        if (lexer->lookahead == '{') {
+          // $${
+          return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+        }
+      }
+      return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK);
+    }
+  }
+
+  if (valid_symbols[TEMPLATE_INTERPOLATION_END] && lexer->lookahead == '}') {
+    scanner_exit_interpolation_context(scanner);
+    return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_END);
+  }
+
+  // handle escape sequences in direct surrounding quoted contexts
+  if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && scanner->quoted_context_depth > 0) {
+    switch (lexer->lookahead) {
+      case '"':
+      case '\n':
+      case '\r':
+      case '\t':
+        return false;
+      case '\\':
+        advance(lexer);
+        switch (lexer->lookahead) {
+          case '"':
+          case 'n':
+          case 'r':
+          case 't':
+          case '\\':
+            return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+          case 'u':
+            for (int i = 0; i < 4; i++) {
+              if (!consume_wxdigit(lexer)) return false;
+            }
+            return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+          case 'U':
+            for (int i = 0; i < 8; i++) {
+              if (!consume_wxdigit(lexer)) return false;
+            }
+            return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+          default:
+            return false;
+        }
+      case '$':
+        // handled above
+      default:
+        return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
+    }
   }
   return false;
 }
 
+void *tree_sitter_hcl_external_scanner_create() {
+  Scanner *scanner = (Scanner*)malloc(sizeof(Scanner));
+  if (scanner) {
+    scanner->in_template_interpolation = false;
+    scanner->in_quoted_context = false;
+    scanner->template_interpolation_depth = 0;
+    scanner->quoted_context_depth = 0;
+  }
+  return scanner;
+}
+
+void tree_sitter_hcl_external_scanner_destroy(void *p) {
+  free(p);
+}
+
+unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) {
+  memcpy(b, p, sizeof(Scanner)); return sizeof(Scanner);
+}
+
+void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b, unsigned n) {
+  memcpy(p, b, n);
+}
+
+bool tree_sitter_hcl_external_scanner_scan(void *p, TSLexer *lexer, const bool *valid_symbols) {
+  return scanner_scan((Scanner*)p, lexer, valid_symbols);
+}
+

test/corpus/attributes.txt

@@ -1,28 +1,32 @@
-==================
+================================================================================
 attribute with literal
-==================
+================================================================================
 
 foo = "bar"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template)))))))
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 attribute with variable
-==================
+================================================================================
 
 foo = bar
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (variable_expr (identifier)))))))
+      (expression
-
+        (variable_expr
+          (identifier))))))

test/corpus/blocks.txt

@@ -1,61 +1,66 @@
-==================
+================================================================================
 basic block
-==================
+================================================================================
 
 block_1 {
 }
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (block
       (identifier))))
 
-==================
+================================================================================
 basic block on one line
-==================
+================================================================================
 
 block_1 {}
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (block
       (identifier))))
 
-==================
+================================================================================
 block with attribute
-==================
+================================================================================
 
 block_1 "strlit1" "strlit2" {
     attr1 = "val1"
 }
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (block
       (identifier)
-      (string_lit)
+      (string_lit
-      (string_lit)
+        (template_literal))
+      (string_lit
+        (template_literal))
       (body
         (attribute
           (identifier)
-          (expression (expr_term (template_expr (quoted_template)))))))))
+          (expression
+            (literal_value
+              (string_lit
+                (template_literal)))))))))
 
-==================
+================================================================================
 nested block
-==================
+================================================================================
 
 block_1 {
   block_2 {
   }
 }
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
@@ -65,15 +70,15 @@ block_1 {
         (block
           (identifier))))))
 
-==================
+================================================================================
 nested block on one line
-==================
+================================================================================
 
 block_1 {
   block_2 {}
 }
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
@@ -83,16 +88,16 @@ block_1 {
         (block
           (identifier))))))
 
-==================
+================================================================================
 nested blocks
-==================
+================================================================================
 
 block_1 {
   block_2 {}
   block_3 {}
 }
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body

test/corpus/collections.txt

@@ -1,43 +1,57 @@
-==================
+================================================================================
 collection value tuple
-==================
+================================================================================
 
 foo = [1, 2, "foo"]
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (collection_value
-          (collection_value
+          (tuple
-            (tuple
+            (expression
-              (expression (expr_term (literal_value (numeric_lit))))
+              (literal_value
-              (expression (expr_term (literal_value (numeric_lit))))
+                (numeric_lit)))
-              (expression (expr_term (template_expr (quoted_template)))))))))))
+            (expression
+              (literal_value
+                (numeric_lit)))
+            (expression
+              (literal_value
+                (string_lit
+                  (template_literal))))))))))
 
-==================
+================================================================================
 collection value object
-==================
+================================================================================
 
 foo = {1: 2, "foo"="bar"}
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (collection_value
-          (collection_value
+          (object
-            (object
+            (object_elem
-              (object_elem
+              (expression
-                (expression (expr_term (literal_value (numeric_lit))))
+                (literal_value
-                (expression (expr_term (literal_value (numeric_lit)))))
+                  (numeric_lit)))
-              (object_elem
+              (expression
-                (expression (expr_term (template_expr (quoted_template))))
+                (literal_value
-                (expression (expr_term (template_expr (quoted_template))))))))))))
+                  (numeric_lit))))
-
+            (object_elem
+              (expression
+                (literal_value
+                  (string_lit
+                    (template_literal))))
+              (expression
+                (literal_value
+                  (string_lit
+                    (template_literal)))))))))))

test/corpus/conditionals.txt

@@ -1,10 +1,10 @@
-==================
+================================================================================
 simple conditional expression
-==================
+================================================================================
 
 foo = predicate() ? 1: 2
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
@@ -12,7 +12,12 @@ foo = predicate() ? 1: 2
       (identifier)
       (expression
         (conditional
-          (expression (expr_term (function_call (identifier))))
+          (expression
-          (expression (expr_term (literal_value (numeric_lit))))
+            (function_call
-          (expression (expr_term (literal_value (numeric_lit)))))))))
+              (identifier)))
-
+          (expression
+            (literal_value
+              (numeric_lit)))
+          (expression
+            (literal_value
+              (numeric_lit))))))))

test/corpus/for_expressions.txt

@@ -1,143 +1,182 @@
-==================
+================================================================================
 for tuple expression
-==================
+================================================================================
 
 foo = [for v in ["a", "b"]: v]
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (for_expr
-          (for_expr
+          (for_tuple_expr
-            (for_tuple_expr
+            (for_intro
-              (for_intro
               (identifier)
               (expression
-                (expr_term
+                (collection_value
-                  (collection_value
+                  (tuple
-                    (tuple
+                    (expression
-                      (expression (expr_term (template_expr (quoted_template))))
+                      (literal_value
-                      (expression (expr_term (template_expr (quoted_template)))))))))
+                        (string_lit
-              (expression (expr_term (variable_expr (identifier)))))))))))
+                          (template_literal))))
+                    (expression
+                      (literal_value
+                        (string_lit
+                          (template_literal))))))))
+            (expression
+              (variable_expr
+                (identifier)))))))))
 
-==================
+================================================================================
 for tuple expression with index
-==================
+================================================================================
 
 foo = [for i, v in ["a", "b"]: i]
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (for_expr
-          (for_expr
+          (for_tuple_expr
-            (for_tuple_expr
+            (for_intro
-              (for_intro
               (identifier)
               (identifier)
               (expression
-                (expr_term
+                (collection_value
-                  (collection_value
+                  (tuple
-                    (tuple
+                    (expression
-                      (expression (expr_term (template_expr (quoted_template))))
+                      (literal_value
-                      (expression (expr_term (template_expr (quoted_template)))))))))
+                        (string_lit
-              (expression (expr_term (variable_expr (identifier)))))))))))
+                          (template_literal))))
+                    (expression
+                      (literal_value
+                        (string_lit
+                          (template_literal))))))))
+            (expression
+              (variable_expr
+                (identifier)))))))))
 
-==================
+================================================================================
 for tuple expression with predicate
-==================
+================================================================================
 
 foo = [for i, v in ["a", "b", "c"]: v if pred(i)]
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-        (expression
+      (expression
-          (expr_term
+        (for_expr
-            (for_expr
+          (for_tuple_expr
-              (for_tuple_expr
+            (for_intro
-                (for_intro
+              (identifier)
-                  (identifier)
+              (identifier)
-                  (identifier)
+              (expression
+                (collection_value
+                  (tuple
                     (expression
-                      (expr_term
+                      (literal_value
-                        (collection_value
+                        (string_lit
-                          (tuple
+                          (template_literal))))
-                            (expression (expr_term (template_expr (quoted_template))))
+                    (expression
-                            (expression (expr_term (template_expr (quoted_template))))
+                      (literal_value
-                            (expression (expr_term (template_expr (quoted_template)))))))))
+                        (string_lit
-                (expression (expr_term (variable_expr (identifier))))
+                          (template_literal))))
-                (for_cond
+                    (expression
-                  (expression
+                      (literal_value
-                    (expr_term
+                        (string_lit
-                      (function_call
+                          (template_literal))))))))
-                        (identifier)
+            (expression
-                        (function_arguments (expression (expr_term (variable_expr (identifier))))))))))))))))
+              (variable_expr
+                (identifier)))
+            (for_cond
+              (expression
+                (function_call
+                  (identifier)
+                  (function_arguments
+                    (expression
+                      (variable_expr
+                        (identifier)))))))))))))
 
-
+================================================================================
-==================
 for object expression
-==================
+================================================================================
 
 foo = {for i, v in ["a", "b"]: v => i}
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (for_expr
-          (for_expr
+          (for_object_expr
-            (for_object_expr
+            (for_intro
-              (for_intro
+              (identifier)
-                (identifier)
+              (identifier)
-                (identifier)
+              (expression
-                  (expression
+                (collection_value
-                    (expr_term
+                  (tuple
-                      (collection_value
+                    (expression
-                        (tuple
+                      (literal_value
-                          (expression (expr_term (template_expr (quoted_template))))
+                        (string_lit
-                          (expression (expr_term (template_expr (quoted_template)))))))))
+                          (template_literal))))
-              (expression (expr_term (variable_expr (identifier))))
+                    (expression
-              (expression (expr_term (variable_expr (identifier)))))))))))
+                      (literal_value
+                        (string_lit
+                          (template_literal))))))))
+            (expression
+              (variable_expr
+                (identifier)))
+            (expression
+              (variable_expr
+                (identifier)))))))))
 
-==================
+================================================================================
 for object expression 2
-==================
+================================================================================
 
 foo = {for i, v in ["a", "b"]: v => i...}
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (for_expr
-          (for_expr
+          (for_object_expr
-            (for_object_expr
+            (for_intro
-              (for_intro
+              (identifier)
-                (identifier)
+              (identifier)
-                (identifier)
+              (expression
-                  (expression
+                (collection_value
-                    (expr_term
+                  (tuple
-                      (collection_value
+                    (expression
-                        (tuple
+                      (literal_value
-                          (expression (expr_term (template_expr (quoted_template))))
+                        (string_lit
-                          (expression (expr_term (template_expr (quoted_template)))))))))
+                          (template_literal))))
-              (expression (expr_term (variable_expr (identifier))))
+                    (expression
-              (expression (expr_term (variable_expr (identifier)))) (ellipsis))))))))
+                      (literal_value
+                        (string_lit
+                          (template_literal))))))))
+            (expression
+              (variable_expr
+                (identifier)))
+            (expression
+              (variable_expr
+                (identifier)))
+            (ellipsis)))))))

test/corpus/function_calls.txt

@@ -1,58 +1,64 @@
-==================
+================================================================================
 nonary function call
-==================
+================================================================================
 
 foo = bar()
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (function_call (identifier)))))))
+      (expression
+        (function_call
+          (identifier))))))
 
-==================
+================================================================================
 unary function call
-==================
+================================================================================
 
 foo = bar("foo")
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (function_call
-          (function_call
+          (identifier)
-            (identifier)
+          (function_arguments
-            (function_arguments
+            (expression
-              (expression (expr_term (template_expr (quoted_template)))))))))))
+              (literal_value
+                (string_lit
+                  (template_literal))))))))))
 
-==================
+================================================================================
 variadic function call
-==================
+================================================================================
 
 foo = bar(x...)
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (function_call
-          (function_call
+          (identifier)
-            (identifier)
+          (function_arguments
-            (function_arguments
+            (expression
-              (expression (expr_term (variable_expr (identifier)))) (ellipsis))))))))
+              (variable_expr
+                (identifier)))
+            (ellipsis)))))))
 
-==================
+================================================================================
 multiline function call
-==================
+================================================================================
 
 foo = bar(
   "a",
@@ -60,18 +66,25 @@ foo = bar(
   "c"
 )
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (function_call
-          (function_call
+          (identifier)
-            (identifier)
+          (function_arguments
-            (function_arguments
+            (expression
-              (expression (expr_term (template_expr (quoted_template))))
+              (literal_value
-              (expression (expr_term (template_expr (quoted_template))))
+                (string_lit
-              (expression (expr_term (template_expr (quoted_template)))))))))))
+                  (template_literal))))
-
+            (expression
+              (literal_value
+                (string_lit
+                  (template_literal))))
+            (expression
+              (literal_value
+                (string_lit
+                  (template_literal))))))))))

test/corpus/literals.txt

@@ -1,128 +1,143 @@
-==================
+================================================================================
 numeric literal scientific notation 1
-==================
+================================================================================
 
 pi = 3
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (numeric_lit)))))))
+      (expression
+        (literal_value
+          (numeric_lit))))))
 
-==================
+================================================================================
 numeric literal scientific notation 2
-==================
+================================================================================
 
 pi = 3.14
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (numeric_lit)))))))
+      (expression
+        (literal_value
+          (numeric_lit))))))
 
-==================
+================================================================================
 numeric literal scientific notation 3
-==================
+================================================================================
 
 big_pi = 3.14e+10
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (numeric_lit)))))))
+      (expression
+        (literal_value
+          (numeric_lit))))))
 
-==================
+================================================================================
 numeric literal scientific notation 4
-==================
+================================================================================
 
 big_pi = 3.14E+10
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (numeric_lit)))))))
+      (expression
+        (literal_value
+          (numeric_lit))))))
 
-==================
+================================================================================
 numeric literal scientific notation 5
-==================
+================================================================================
 
 small_pi = 3.14e-10
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (numeric_lit)))))))
+      (expression
+        (literal_value
+          (numeric_lit))))))
 
-
+================================================================================
-==================
 numeric literal scientific notation 6
-==================
+================================================================================
 
 small_pi = 3.14E-10
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (numeric_lit)))))))
+      (expression
+        (literal_value
+          (numeric_lit))))))
 
-==================
+================================================================================
 bool literal true
-==================
+================================================================================
 
 foo = true
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (bool_lit)))))))
+      (expression
+        (literal_value
+          (bool_lit))))))
 
-==================
+================================================================================
 bool literal false
-==================
+================================================================================
 
 foo = false
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (bool_lit)))))))
+      (expression
+        (literal_value
+          (bool_lit))))))
 
-==================
+================================================================================
 null literal
-==================
+================================================================================
 
 foo = null
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (literal_value (null_lit)))))))
+      (expression
-
+        (literal_value
-
+          (null_lit))))))

test/corpus/operators.txt

@@ -1,10 +1,10 @@
-==================
+================================================================================
 unary operator -
-==================
+================================================================================
 
 foo = -3
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
@@ -12,39 +12,44 @@ foo = -3
       (identifier)
       (expression
         (operation
-          (unary_operation (expr_term (literal_value (numeric_lit)))))))))
+          (unary_operation
+            (literal_value
+              (numeric_lit))))))))
 
-==================
+================================================================================
 unary operator !
-==================
+================================================================================
 
 foo = !true
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-        (expression
+      (expression
-          (operation
+        (operation
-            (unary_operation (expr_term (literal_value (bool_lit)))))))))
+          (unary_operation
+            (literal_value
+              (bool_lit))))))))
 
-==================
+================================================================================
 binary operators +
-==================
+================================================================================
 
 foo = 1+2
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-        (expression
+      (expression
-          (operation
+        (operation
-            (binary_operation
+          (binary_operation
-              (expr_term (literal_value (numeric_lit)))
+            (literal_value
-              (expr_term (literal_value (numeric_lit)))))))))
+              (numeric_lit))
-
+            (literal_value
+              (numeric_lit))))))))

test/corpus/splat.txt

@@ -1,86 +1,77 @@
-==================
+================================================================================
 get attr
-==================
+================================================================================
 
 foo = bar.baz
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (variable_expr
-          (expr_term
+          (identifier))
-            (variable_expr
+        (get_attr
-              (identifier)))
+          (identifier))))))
-          (get_attr
-            (identifier)))))))
 
-==================
+================================================================================
 get index
-==================
+================================================================================
 
 foo = bar[1]
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (variable_expr
-          (expr_term
+          (identifier))
-            (variable_expr
+        (index
-              (identifier)))
+          (expression
-          (index
+            (literal_value
-            (expression
+              (numeric_lit))))))))
-              (expr_term
-                (literal_value
-                  (numeric_lit))))))))))
 
-==================
+================================================================================
 attr splat
-==================
+================================================================================
 
 foo = bar.*.foo
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (variable_expr
-          (expr_term
+          (identifier))
-            (variable_expr
-              (identifier)))
         (splat
-          (attr_splat
+          (attr_splat))
-            (get_attr
+        (get_attr
-              (identifier)))))))))
+          (identifier))))))
 
-==================
+================================================================================
 full splat
-==================
+================================================================================
 
 foo = bar[*].foo
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
       (expression
-        (expr_term
+        (variable_expr
-          (expr_term
+          (identifier))
-            (variable_expr
-              (identifier)))
         (splat
-          (full_splat
+          (full_splat))
-            (get_attr
+        (get_attr
-              (identifier)))))))))
+          (identifier))))))

test/corpus/strings.txt

@@ -1,303 +1,229 @@
-==================
+================================================================================
-string literal one line
-==================
-
-foo = "bar"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template)))))))
-
-==================
-string literal escaped newline
-==================
-
-foo = "bar\nbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
-
-==================
-string literal escaped tab
-==================
-
-foo = "bar\tbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
-
-==================
-string literal escaped "
-==================
-
-foo = "bar\"baz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
-
-==================
-string literal escaped \
-==================
-
-foo = "bar\\baz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
-
-==================
-string literal escaped \uFFFF
-==================
-
-foo = "bar\uFFFFbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
-
-==================
-string bad escape sequence
-==================
-
-foo = "bar\pbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED '\')))))))))
-
-==================
 string bad escape sequence 2
-==================
+================================================================================
 
 foo = "bar\uZZ"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED '\')))))))))
+      (expression
+        (template_expr
+          (quoted_template
+            (template_literal
+              (ERROR
+                (UNEXPECTED '\')))))))))
 
-==================
+================================================================================
 string literal multi line error
-==================
+================================================================================
 
 foo = "
 bar"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED 'b')))))))))
+      (expression
+        (template_expr
+          (quoted_template
+            (ERROR
+              (UNEXPECTED 'b'))
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal unescaped tab
-==================
+================================================================================
 
 foo = "foo	bar"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED 'b')))))))))
+      (expression
+        (template_expr
+          (quoted_template
+            (template_literal
+              (ERROR
+                (UNEXPECTED 'b')))))))))
 
-==================
+================================================================================
 string literal unescaped backslash
-==================
+================================================================================
 
 foo = "foo\bar"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED '\')))))))))
+      (expression
+        (template_expr
+          (quoted_template
+            (template_literal
+              (ERROR
+                (UNEXPECTED '\')))))))))
 
-==================
+================================================================================
 string literal escaped backslash at end
-==================
+================================================================================
 
 foo = "foo\\"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))
 
-
+================================================================================
-==================
 string literal escaped template interpolation
-==================
+================================================================================
 
 foo = "$${foo.bar}"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal template chars but no template 1
-==================
+================================================================================
 
 foo = "$$$"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template)))))))
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal template chars but no template 2
-==================
+================================================================================
 
 foo = "100%"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template)))))))
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal template chars but no template 3
-==================
+================================================================================
 
 foo = "%\n\t"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template
+      (expression
-        (escape_sequence)
+        (literal_value
-        (escape_sequence))))))))
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal template chars but no template 4
-==================
+================================================================================
 
 foo = "%%\n\t"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template
+      (expression
-        (escape_sequence)
+        (literal_value
-        (escape_sequence))))))))
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal template chars but no template 5
-==================
+================================================================================
 
 foo = "$$"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template)))))))
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal template chars but no template 6
-==================
+================================================================================
 
 foo = "%%{\n\t"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template
+      (expression
-        (escape_sequence)
+        (literal_value
-        (escape_sequence)
+          (string_lit
-        (escape_sequence))))))))
+            (template_literal)))))))
 
-==================
+================================================================================
 string literal escaped template
-==================
+================================================================================
 
 foo = "$${ var.bar }"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+      (expression
-
+        (literal_value
-==================
+          (string_lit
-proper quoted template in string literal position errors
+            (template_literal)))))))
-==================
-
-resource "${var.bar}" {
-}
-
----
-
-(config_file
-  (body
-    (block
-      (identifier)
-      (string_lit (ERROR)))))

test/corpus/templates.txt

@@ -1,13 +1,145 @@
-==================
+================================================================================
-quoted template expression
+simple quoted template expression
-==================
+================================================================================
 
 foo = "${ var.bar }"
 
----
+--------------------------------------------------------------------------------
 
 (config_file
   (body
     (attribute
       (identifier)
-      (expression (expr_term (template_expr (quoted_template (template_interpolation))))))))
+      (expression
+        (template_expr
+          (quoted_template
+            (template_interpolation
+              (expression
+                (variable_expr
+                  (identifier))
+                (get_attr
+                  (identifier))))))))))
+
+================================================================================
+quoted template expression with escaped strings and fake strip marker
+================================================================================
+
+foo = "${ " ~ " }"
+
+--------------------------------------------------------------------------------
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression
+        (template_expr
+          (quoted_template
+            (template_interpolation
+              (expression
+                (literal_value
+                  (string_lit
+                    (template_literal)))))))))))
+
+================================================================================
+quoted template with nested quoted template
+================================================================================
+
+foo = "p-${ foo("v-${a.b}") }"
+
+--------------------------------------------------------------------------------
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression
+        (template_expr
+          (quoted_template
+            (template_literal)
+            (template_interpolation
+              (expression
+                (function_call
+                  (identifier)
+                  (function_arguments
+                    (expression
+                      (template_expr
+                        (quoted_template
+                          (template_literal)
+                          (template_interpolation
+                            (expression
+                              (variable_expr
+                                (identifier))
+                              (get_attr
+                                (identifier)))))))))))))))))
+
+================================================================================
+quoted template interpolation with strip markers
+================================================================================
+
+foo = "hello ${~ "world" ~} hello"
+
+--------------------------------------------------------------------------------
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression
+        (template_expr
+          (quoted_template
+            (template_literal)
+            (template_interpolation
+              (strip_marker)
+              (expression
+                (literal_value
+                  (string_lit
+                    (template_literal))))
+              (strip_marker))
+            (template_literal)))))))
+
+================================================================================
+quoted template object expression in template
+================================================================================
+
+foo = "${ {a=b}[a] }"
+
+--------------------------------------------------------------------------------
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression
+        (template_expr
+          (quoted_template
+            (template_interpolation
+              (expression
+                (collection_value
+                  (object
+                    (object_elem
+                      (identifier)
+                      (expression
+                        (variable_expr
+                          (identifier))))))
+                (index
+                  (expression
+                    (variable_expr
+                      (identifier))))))))))))
+
+================================================================================
+escaped template interpolation start
+================================================================================
+
+foo = "hello $${ world"
+
+--------------------------------------------------------------------------------
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression
+        (literal_value
+          (string_lit
+            (template_literal)))))))