Merge pull request #1 from MichaHoffmann/add_template_expressions

add quoted_templates and template interpolations

.github/workflows/unittests.yaml

@@ -1,7 +1,6 @@
 name: unittests
 on:
   push:
-  pull_request:
 
 jobs:
   unittests:

.gitignore

@@ -1,4 +1,6 @@
 node_modules
+queries
+package-lock.json
 build
 npm-debug.log
 .env

README.md

@@ -2,15 +2,37 @@
 
 tree-sitter grammar for the [HCL](https://github.com/hashicorp/hcl/blob/main/hclsyntax/spec.md) language
 
-## developing
+## Developing
 
 It is recommended to use `nix` to fulfill all development dependencies. To activate the development environment simply run `nix-shell` in the project root.
 
-## running tests
+## Running Tests
 
 To run tests simply run `nix-shell --run 'tree-sitter test'`. 
 
-## todo
+## Example
 
-* use [Unicode® Standard Annex #31](https://www.unicode.org/reports/tr31/) (augmented with '-')for identifiers
+Highlighting `example/example.hcl`:
+
+![Highlighting Example](https://i.imgur.com/JdfxHvE.jpeg)
+
+## Quoted Template Expressions
+
+In principle it is allowed to contain arbitary expressions in quoted template expressions.
+Consider for example:
+
+```hcl
+foo = "prefix-${func(\"bar\"}"
+```
+
+To make parsing a little easier, this parser only checks for valid escape sequences and template chars. 
+When using this parser one would have to take the content of a template interpolation, unescape it and parse it again to get the syntax tree. The same applies to template directives.
+
+## String Literals
+
+String literals are parsed as quoted templates. The calling application should check if the node contains any template interpolations or directives.
+
+## Todo
+
+* use [Unicode® Standard Annex #31](https://www.unicode.org/reports/tr31/) (augmented with '-') for identifiers
 * add [template expressions](https://github.com/hashicorp/hcl/blob/main/hclsyntax/spec.md#template-expressions) 

example/example.hcl

@@ -15,15 +15,18 @@ resource_1 "strlit1" "strlit2" {
   splat1 = tuple.*.foo.bar[0]
   splat2 = tuple[*].foo.bar[0]
   for1   = { for i, v in ["a", "a", "b"] : v => i... }
-  for2   = [ for k, v in local.map : "${k}-${v}" ]
+  for2   = [ for k, v in var.map : "${k}-${v}" ]
-  for3   = { for k, v in local.map : k => v }
+  for3   = { for k, v in var.map : k => v }
-  for4   = [ for v in local.list : v ]
+  for4   = [ for v in var.list : v ]
-  for5   = { for v in local.list : v => v }
+  for5   = { for v in var.list : v => v }
-  for6   = [ for v in local.list : v if v < 3 ]
+  for6   = [ for v in var.list : v if v < 3 ]
   func1  = is_number("123")
   cond1  = (1 == 2) ? 1 : "foobar"
   bin1   = ((1+2)%3)*4
   esc1   = "\" \t \UFF11FF22 \uFFFF \n"
+  esc2   = "$${} %%{}"
+  tpl1   = "prefix-${var.bar}"
+  tpl2   = "prefix-${func(\"bar\"}"
 
   nested_resource_1 {
     attr1 = 2

grammar.js

@@ -1,4 +1,16 @@
 const
+  PREC = {
+    unary: 7,
+    binary_mult: 6,
+    binary_add: 5,
+    binary_ord: 4,
+    binary_comp: 3,
+    binary_and: 2,
+    binary_or: 1,
+
+    quoted_string: 2,
+    quoted_template: 1,
+  }
   unicodeLetter = /\p{L}/
   unicodePunctuation = /\p{Pc}/
   unicodeDigit = /[0-9]/
@@ -7,11 +19,17 @@ module.exports = grammar({
   name: 'hcl',
 
   conflicts: $ => [
-      [$.body],
+    [$.body],
-      [$.object_elem, $.variable_expr],
+    [$.object_elem, $.variable_expr],
-      [$.attr_splat],
+    [$.attr_splat],
-      [$.full_splat],
+    [$.full_splat],
-      [$.conditional],
+    [$.conditional],
+  ],
+
+  externals: $ => [
+    $._template_char,
+    $._template_char_in_interpolation,
+    $.escape_sequence,
   ],
 
   extras: $ => [
@@ -57,7 +75,7 @@ module.exports = grammar({
 
     expr_term: $ => choice(
       $.literal_value,
-      // $.template_expr,
+      $.template_expr,
       $.collection_value,
       $.variable_expr,
       $.function_call,
@@ -70,36 +88,18 @@ module.exports = grammar({
 
     literal_value: $ => choice(
       $.numeric_lit,
-      $.string_lit,
       $.bool_lit,
       $.null_lit,
     ),
 
     numeric_lit: $ => /[0-9]+(\.[0-9]+([eE][-+]?[0-9]+)?)?/,
 
-    string_lit: $ => seq(
-      '"',
-      repeat(choice(token.immediate(prec(1, /[^\\"\n\r\t]+/)), $.escape_sequence)),
-      '"',
-    ),
-
-    escape_sequence: $ => token.immediate(seq(
-      '\\',
-      choice(
-        '\\',
-        '"',
-        'n',
-        'r',
-        't',
-        /u[0-9a-fA-F]{4}/,
-        /U[0-9a-fA-F]{8}/
-      )
-    )),
-
     bool_lit: $ => choice('true', 'false'),
 
     null_lit: $ => 'null',
 
+    // string_lit is defined as quoted template
+
     collection_value: $ => choice(
       $.tuple,
       $.object,
@@ -210,16 +210,16 @@ module.exports = grammar({
 
     operation: $ => choice($.unary_operation, $.binary_operation),
 
-    unary_operation: $ => prec.left(7, seq(choice('-', '!'), $.expr_term)),
+    unary_operation: $ => prec.left(PREC.unary, seq(choice('-', '!'), $.expr_term)),
 
     binary_operation: $ => {
       const table = [
-        [6, choice('*', '/', '%')],
+        [PREC.binary_mult, choice('*', '/', '%')],
-        [5, choice('+', '-')],
+        [PREC.binary_add, choice('+', '-')],
-        [4, choice('>', '>=', '<', '<=')],
+        [PREC.binary_ord, choice('>', '>=', '<', '<=')],
-        [3, choice('==', '!=')],
+        [PREC.binary_comp, choice('==', '!=')],
-        [2, choice('&&')],
+        [PREC.binary_and, choice('&&')],
-        [1, choice('||')],
+        [PREC.binary_or, choice('||')],
       ];
 
       return choice(...table.map(([precedence, operator]) =>
@@ -228,6 +228,30 @@ module.exports = grammar({
       );
     },
 
+    template_expr: $ => choice(
+      $.quoted_template,
+      // $.heredoc_template,
+    ),
+
+    // application should check that no template interpolation is contained
+    string_lit: $ => $.quoted_template,
+
+    quoted_template: $ => seq(
+      '"', 
+      repeat(choice(
+        $._template_char, 
+        $.escape_sequence,
+        $.template_interpolation,
+      )), 
+      '"',
+    ),
+
+    template_interpolation: $ => seq(
+      choice('${', '${~'),
+      repeat(choice($._template_char_in_interpolation, $.escape_sequence)),
+      choice('}', '~}'),
+    ),
+
     // http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890
     comment: $ => token(choice(
       seq('#', /.*/),

package-lock.json

@@ -1,13 +0,0 @@
-{
-  "name": "tree-sitter-hcl",
-  "version": "1.0.0",
-  "lockfileVersion": 1,
-  "requires": true,
-  "dependencies": {
-    "tree-sitter-cli": {
-      "version": "0.19.5",
-      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.19.5.tgz",
-      "integrity": "sha512-kRzKrUAwpDN9AjA3b0tPBwT1hd8N2oQvvvHup2OEsX6mdsSMLmAvR+NSqK9fe05JrRbVvG8mbteNUQsxlMQohQ=="
-    }
-  }
-}

package.json

@@ -1,11 +1,8 @@
 {
   "name": "tree-sitter-hcl",
-  "version": "1.0.0",
+  "version": "0.2.0-snapshot",
   "description": "",
   "main": "bindings/node",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
   "author": "",
   "license": "ISC",
   "dependencies": {

src/grammar.json

@@ -152,6 +152,10 @@
           "type": "SYMBOL",
           "name": "literal_value"
         },
+        {
+          "type": "SYMBOL",
+          "name": "template_expr"
+        },
         {
           "type": "SYMBOL",
           "name": "collection_value"
@@ -233,10 +237,6 @@
           "type": "SYMBOL",
           "name": "numeric_lit"
         },
-        {
-          "type": "SYMBOL",
-          "name": "string_lit"
-        },
         {
           "type": "SYMBOL",
           "name": "bool_lit"
@@ -251,87 +251,6 @@
       "type": "PATTERN",
       "value": "[0-9]+(\\.[0-9]+([eE][-+]?[0-9]+)?)?"
     },
-    "string_lit": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "\""
-        },
-        {
-          "type": "REPEAT",
-          "content": {
-            "type": "CHOICE",
-            "members": [
-              {
-                "type": "IMMEDIATE_TOKEN",
-                "content": {
-                  "type": "PREC",
-                  "value": 1,
-                  "content": {
-                    "type": "PATTERN",
-                    "value": "[^\\\\\"\\n\\r\\t]+"
-                  }
-                }
-              },
-              {
-                "type": "SYMBOL",
-                "name": "escape_sequence"
-              }
-            ]
-          }
-        },
-        {
-          "type": "STRING",
-          "value": "\""
-        }
-      ]
-    },
-    "escape_sequence": {
-      "type": "IMMEDIATE_TOKEN",
-      "content": {
-        "type": "SEQ",
-        "members": [
-          {
-            "type": "STRING",
-            "value": "\\"
-          },
-          {
-            "type": "CHOICE",
-            "members": [
-              {
-                "type": "STRING",
-                "value": "\\"
-              },
-              {
-                "type": "STRING",
-                "value": "\""
-              },
-              {
-                "type": "STRING",
-                "value": "n"
-              },
-              {
-                "type": "STRING",
-                "value": "r"
-              },
-              {
-                "type": "STRING",
-                "value": "t"
-              },
-              {
-                "type": "PATTERN",
-                "value": "u[0-9a-fA-F]{4}"
-              },
-              {
-                "type": "PATTERN",
-                "value": "U[0-9a-fA-F]{8}"
-              }
-            ]
-          }
-        ]
-      }
-    },
     "bool_lit": {
       "type": "CHOICE",
       "members": [
@@ -1086,6 +1005,99 @@
         }
       ]
     },
+    "template_expr": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "quoted_template"
+        }
+      ]
+    },
+    "string_lit": {
+      "type": "SYMBOL",
+      "name": "quoted_template"
+    },
+    "quoted_template": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "\""
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "_template_char"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "escape_sequence"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "template_interpolation"
+              }
+            ]
+          }
+        },
+        {
+          "type": "STRING",
+          "value": "\""
+        }
+      ]
+    },
+    "template_interpolation": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "STRING",
+              "value": "${"
+            },
+            {
+              "type": "STRING",
+              "value": "${~"
+            }
+          ]
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "_template_char_in_interpolation"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "escape_sequence"
+              }
+            ]
+          }
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "STRING",
+              "value": "}"
+            },
+            {
+              "type": "STRING",
+              "value": "~}"
+            }
+          ]
+        }
+      ]
+    },
     "comment": {
       "type": "TOKEN",
       "content": {
@@ -1167,7 +1179,20 @@
     ]
   ],
   "precedences": [],
-  "externals": [],
+  "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "_template_char"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_template_char_in_interpolation"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "escape_sequence"
+    }
+  ],
   "inline": [],
   "supertypes": []
 }

src/node-types.json

@@ -206,6 +206,10 @@
           "type": "splat",
           "named": true
         },
+        {
+          "type": "template_expr",
+          "named": true
+        },
         {
           "type": "variable_expr",
           "named": true
@@ -452,10 +456,6 @@
         {
           "type": "numeric_lit",
           "named": true
-        },
-        {
-          "type": "string_lit",
-          "named": true
         }
       ]
     }
@@ -513,6 +513,25 @@
       ]
     }
   },
+  {
+    "type": "quoted_template",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "escape_sequence",
+          "named": true
+        },
+        {
+          "type": "template_interpolation",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "splat",
     "named": true,
@@ -536,6 +555,36 @@
     "type": "string_lit",
     "named": true,
     "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "quoted_template",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "template_expr",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "quoted_template",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "template_interpolation",
+    "named": true,
+    "fields": {},
     "children": {
       "multiple": true,
       "required": false,
@@ -604,6 +653,14 @@
     "type": "\"",
     "named": false
   },
+  {
+    "type": "${",
+    "named": false
+  },
+  {
+    "type": "${~",
+    "named": false
+  },
   {
     "type": "%",
     "named": false
@@ -743,5 +800,9 @@
   {
     "type": "}",
     "named": false
+  },
+  {
+    "type": "~}",
+    "named": false
   }
 ]

src/scanner.c

@@ -0,0 +1,148 @@
+#include <tree_sitter/parser.h>
+#include <wctype.h>
+
+// forward declarations
+
+enum TokenType {
+  TEMPLATE_CHAR,
+  TEMPLATE_CHAR_IN_INTERPOLATION,
+  ESCAPE_SEQUENCE
+};
+
+// helpers
+
+static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+
+static bool accept_template_char_inplace(TSLexer *lexer) {
+    lexer->result_symbol = TEMPLATE_CHAR;
+    return true;
+}
+
+static bool advance_and_accept_template_char(TSLexer *lexer) {
+    advance(lexer);
+    return accept_template_char_inplace(lexer);
+}
+
+static bool advance_and_accept_escape_sequence(TSLexer *lexer) {
+    lexer->result_symbol = ESCAPE_SEQUENCE;
+    advance(lexer);
+    return true;
+}
+
+static bool advance_and_accept_template_char_in_interpolation(TSLexer *lexer) {
+    advance(lexer);
+    lexer->result_symbol = TEMPLATE_CHAR_IN_INTERPOLATION;
+    return true;
+}
+
+static bool consume_wxdigit(TSLexer *lexer) {
+    advance(lexer);
+    return iswxdigit(lexer->lookahead);
+}
+
+// scan escape sequences \n \t \r \\ \" \uHHHH or \UHHHHHHHHA where H is a hex digit
+// assumes that the leading character is '\'
+static bool scan_backslash_escape_sequence(TSLexer *lexer) {
+  advance(lexer);
+  switch (lexer->lookahead) {
+    case '"':
+    case 'n':
+    case 'r':
+    case 't':
+    case '\\':
+      return advance_and_accept_escape_sequence(lexer);
+    case 'u':
+      for (int i = 0; i < 4; i++) {
+        if (!consume_wxdigit(lexer)) return false;
+      }
+      return advance_and_accept_escape_sequence(lexer);
+    case 'U':
+      for (int i = 0; i < 8; i++) {
+        if (!consume_wxdigit(lexer)) return false;
+      }
+      return advance_and_accept_escape_sequence(lexer);
+    default:
+      return false;
+  }
+}
+
+// may accept multiple characters like %% as a 'template literal chunk'
+// assumes that the leading character is '%' or '$'
+static bool scan_template_literal_chunk_or_template_escape_sequence(TSLexer *lexer) {
+  const leading_char = lexer->lookahead;
+
+  advance(lexer);
+  // reject %{ because its the start of template directives
+  if (lexer->lookahead == '{') return false;
+  if (lexer->lookahead == leading_char) {
+    advance(lexer);
+    // accept %%{ as escape sequence
+    if (lexer->lookahead == '{') return advance_and_accept_escape_sequence(lexer);
+  }
+  // accept % and %% as template chars
+  return accept_template_char_inplace(lexer);
+}
+
+static bool scan_template_char_or_escape_sequence(TSLexer *lexer) {
+  switch (lexer->lookahead) {
+    case '"':
+    case '\n':
+    case '\r':
+    case '\t':
+      return false;
+    case '\\':
+      return scan_backslash_escape_sequence(lexer);
+    case '$':
+    case '%':
+      return scan_template_literal_chunk_or_template_escape_sequence(lexer);
+    default:
+      return advance_and_accept_template_char(lexer);
+  }
+}
+
+static bool scan_template_char_in_interpolation_or_escape_sequence(TSLexer *lexer) {
+  switch (lexer->lookahead) {
+    case '"':
+    case '\n':
+    case '\r':
+    case '\t':
+    // no template interpolation chars are allowed in template interpolations ( even escaped )
+    case '$':
+    case '%':
+    case '~':
+      return false;
+    // '}' ends the template interpolation
+    case '}':
+      return false;
+    case '\\':
+      return scan_backslash_escape_sequence(lexer);
+    default:
+      return advance_and_accept_template_char_in_interpolation(lexer);
+  }
+}
+
+// scanner
+
+void *tree_sitter_hcl_external_scanner_create() { return NULL; }
+void tree_sitter_hcl_external_scanner_destroy(void *p) {}
+void tree_sitter_hcl_external_scanner_reset(void *p) {}
+unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) { return 0; }
+void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
+
+
+bool tree_sitter_hcl_external_scanner_scan(
+  void *p,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  // when scanning string literals or quoted template literals that are not in an template directive
+  if (valid_symbols[TEMPLATE_CHAR] && valid_symbols[ESCAPE_SEQUENCE]) {
+    return scan_template_char_or_escape_sequence(lexer);
+  }
+  // quoted template literals currently inside a template directive
+  if (valid_symbols[TEMPLATE_CHAR_IN_INTERPOLATION] && valid_symbols[ESCAPE_SEQUENCE]) {
+    return scan_template_char_in_interpolation_or_escape_sequence(lexer);
+  }
+  return false;
+}
+

test/corpus/attributes.txt

@@ -10,10 +10,7 @@ foo = "bar"
   (body
     (attribute
       (identifier)
-      (expression
+      (expression (expr_term (template_expr (quoted_template)))))))
-        (expr_term
-          (literal_value
-            (string_lit)))))))
 
 ==================
 attribute with variable
@@ -27,8 +24,5 @@ foo = bar
   (body
     (attribute
       (identifier)
-      (expression
+      (expression (expr_term (variable_expr (identifier)))))))
-        (expr_term
-          (variable_expr
-            (identifier)))))))
 

test/corpus/blocks.txt

@@ -39,15 +39,12 @@ block_1 "strlit1" "strlit2" {
   (body
     (block
       (identifier)
-      (string_lit)
+      (string_lit (quoted_template))
-      (string_lit)
+      (string_lit (quoted_template))
       (body
         (attribute
           (identifier)
-          (expression
+          (expression (expr_term (template_expr (quoted_template)))))))))
-            (expr_term
-              (literal_value
-                (string_lit)))))))))
 
 ==================
 nested block

test/corpus/collections.txt

@@ -16,7 +16,7 @@ foo = [1, 2, "foo"]
             (tuple 
               (expression (expr_term (literal_value (numeric_lit)))) 
               (expression (expr_term (literal_value (numeric_lit)))) 
-              (expression (expr_term (literal_value (string_lit)))))))))))
+              (expression (expr_term (template_expr (quoted_template)))))))))))
 
 ==================
 collection value object
@@ -38,6 +38,6 @@ foo = {1: 2, "foo"="bar"}
                 (expression (expr_term (literal_value (numeric_lit)))) 
                 (expression (expr_term (literal_value (numeric_lit))))) 
               (object_elem 
-                (expression (expr_term (literal_value (string_lit)))) 
+                (expression (expr_term (template_expr (quoted_template)))) 
-                (expression (expr_term (literal_value (string_lit)))))))))))) 
+                (expression (expr_term (template_expr (quoted_template)))))))))))) 
 

test/corpus/for_expressions.txt

@@ -20,8 +20,8 @@ foo = [for v in ["a", "b"]: v]
                 (expr_term 
                   (collection_value 
                     (tuple 
-                      (expression (expr_term (literal_value (string_lit)))) 
+                      (expression (expr_term (template_expr (quoted_template)))) 
-                      (expression (expr_term (literal_value (string_lit)))))))))
+                      (expression (expr_term (template_expr (quoted_template)))))))))
               (expression (expr_term (variable_expr (identifier))))))))))) 
 
 ==================
@@ -47,8 +47,8 @@ foo = [for i, v in ["a", "b"]: i]
                 (expr_term 
                   (collection_value 
                     (tuple 
-                      (expression (expr_term (literal_value (string_lit)))) 
+                      (expression (expr_term (template_expr (quoted_template)))) 
-                      (expression (expr_term (literal_value (string_lit)))))))))
+                      (expression (expr_term (template_expr (quoted_template)))))))))
               (expression (expr_term (variable_expr (identifier))))))))))) 
 
 ==================
@@ -74,9 +74,9 @@ foo = [for i, v in ["a", "b", "c"]: v if pred(i)]
                       (expr_term 
                         (collection_value 
                           (tuple 
-                            (expression (expr_term (literal_value (string_lit)))) 
+                            (expression (expr_term (template_expr (quoted_template)))) 
-                            (expression (expr_term (literal_value (string_lit)))) 
+                            (expression (expr_term (template_expr (quoted_template)))) 
-                            (expression (expr_term (literal_value (string_lit))))))))) 
+                            (expression (expr_term (template_expr (quoted_template))))))))) 
                 (expression (expr_term (variable_expr (identifier)))) 
                 (for_cond 
                   (expression 
@@ -109,8 +109,8 @@ foo = {for i, v in ["a", "b"]: v => i}
                     (expr_term 
                       (collection_value 
                         (tuple 
-                          (expression (expr_term (literal_value (string_lit)))) 
+                          (expression (expr_term (template_expr (quoted_template)))) 
-                          (expression (expr_term (literal_value (string_lit))))))))) 
+                          (expression (expr_term (template_expr (quoted_template))))))))) 
               (expression (expr_term (variable_expr (identifier)))) 
               (expression (expr_term (variable_expr (identifier))))))))))) 
 
@@ -137,7 +137,7 @@ foo = {for i, v in ["a", "b"]: v => i...}
                     (expr_term 
                       (collection_value 
                         (tuple 
-                          (expression (expr_term (literal_value (string_lit)))) 
+                          (expression (expr_term (template_expr (quoted_template)))) 
-                          (expression (expr_term (literal_value (string_lit))))))))) 
+                          (expression (expr_term (template_expr (quoted_template))))))))) 
               (expression (expr_term (variable_expr (identifier)))) 
               (expression (expr_term (variable_expr (identifier)))) (ellipsis)))))))) 

test/corpus/function_calls.txt

@@ -10,10 +10,7 @@ foo = bar()
   (body
     (attribute
       (identifier)
-      (expression
+      (expression (expr_term (function_call (identifier)))))))
-        (expr_term
-          (function_call
-            (identifier)))))))
 
 ==================
 unary function call
@@ -32,10 +29,7 @@ foo = bar("foo")
           (function_call 
             (identifier) 
             (function_arguments 
-              (expression 
+              (expression (expr_term (template_expr (quoted_template))))))))))) 
-                (expr_term 
-                  (literal_value 
-                    (string_lit))))))))))) 
 
 ==================
 variadic function call
@@ -77,7 +71,7 @@ foo = bar(
           (function_call 
             (identifier) 
             (function_arguments 
-              (expression (expr_term (literal_value (string_lit)))) 
+              (expression (expr_term (template_expr (quoted_template)))) 
-              (expression (expr_term (literal_value (string_lit)))) 
+              (expression (expr_term (template_expr (quoted_template)))) 
-              (expression (expr_term (literal_value (string_lit))))))))))) 
+              (expression (expr_term (template_expr (quoted_template))))))))))) 
 

test/corpus/literals.txt

@@ -125,158 +125,4 @@ foo = null
       (identifier)
       (expression (expr_term (literal_value (null_lit)))))))
 
-==================
-string literal one line
-==================
-
-foo = "bar"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit)))))))
-
-==================
-string literal escaped newline
-==================
-
-foo = "bar\nbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (escape_sequence))))))))
-
-==================
-string literal escaped tab
-==================
-
-foo = "bar\tbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (escape_sequence))))))))
-
-==================
-string literal escaped "
-==================
-
-foo = "bar\"baz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (escape_sequence))))))))
-
-==================
-string literal escaped \
-==================
-
-foo = "bar\\baz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (escape_sequence))))))))
-
-==================
-string literal escaped \uFFFF
-==================
-
-foo = "bar\uFFFFbaz"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (escape_sequence))))))))
-
-==================
-string bad escape sequence
-==================
-
-foo = "bar\pbaz"
-
----
-
-(config_file 
-  (body 
-    (attribute 
-      (identifier) 
-      (expression (expr_term (literal_value (string_lit (ERROR (UNEXPECTED 'p'))))))))) 
-
-==================
-string bad escape sequence 2
-==================
-
-foo = "bar\uZZ"
-
----
-
-(config_file 
-  (body 
-    (attribute 
-      (identifier) 
-      (expression (expr_term (literal_value (string_lit (ERROR (UNEXPECTED 'Z'))))))))) 
-
-==================
-string literal multi line error
-==================
-
-foo = "
-bar"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (ERROR (UNEXPECTED 'b')))))))))
-
-==================
-string literal unescaped tab
-==================
-
-foo = "foo	bar"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (ERROR (UNEXPECTED 'b')))))))))
-
-==================
-string literal unescaped backslash
-==================
-
-foo = "foo\bar"
-
----
-
-(config_file
-  (body
-    (attribute
-      (identifier)
-      (expression (expr_term (literal_value (string_lit (ERROR (UNEXPECTED 'b')))))))))
 

test/corpus/strings.txt

@@ -0,0 +1,288 @@
+==================
+string literal one line
+==================
+
+foo = "bar"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template)))))))
+
+==================
+string literal escaped newline
+==================
+
+foo = "bar\nbaz"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+==================
+string literal escaped tab
+==================
+
+foo = "bar\tbaz"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+==================
+string literal escaped "
+==================
+
+foo = "bar\"baz"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+==================
+string literal escaped \
+==================
+
+foo = "bar\\baz"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+==================
+string literal escaped \uFFFF
+==================
+
+foo = "bar\uFFFFbaz"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+==================
+string bad escape sequence
+==================
+
+foo = "bar\pbaz"
+
+---
+
+(config_file 
+  (body 
+    (attribute 
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED '\'))))))))) 
+
+==================
+string bad escape sequence 2
+==================
+
+foo = "bar\uZZ"
+
+---
+
+(config_file 
+  (body 
+    (attribute 
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED '\'))))))))) 
+
+==================
+string literal multi line error
+==================
+
+foo = "
+bar"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED 'b')))))))))
+
+==================
+string literal unescaped tab
+==================
+
+foo = "foo	bar"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED 'b')))))))))
+
+==================
+string literal unescaped backslash
+==================
+
+foo = "foo\bar"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (ERROR (UNEXPECTED '\')))))))))
+
+==================
+string literal escaped backslash at end
+==================
+
+foo = "foo\\"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+
+==================
+string literal escaped template interpolation
+==================
+
+foo = "$${foo.bar}"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))
+
+==================
+string literal template chars but no template 1
+==================
+
+foo = "$$$"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template)))))))
+
+==================
+string literal template chars but no template 2
+==================
+
+foo = "100%"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template)))))))
+
+==================
+string literal template chars but no template 3
+==================
+
+foo = "%\n\t"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template 
+        (escape_sequence) 
+        (escape_sequence)))))))) 
+
+==================
+string literal template chars but no template 4
+==================
+
+foo = "%%\n\t"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template 
+        (escape_sequence) 
+        (escape_sequence)))))))) 
+
+==================
+string literal template chars but no template 5
+==================
+
+foo = "$$"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template))))))) 
+
+==================
+string literal template chars but no template 6
+==================
+
+foo = "%%{\n\t"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template 
+        (escape_sequence) 
+        (escape_sequence) 
+        (escape_sequence)))))))) 
+
+==================
+string literal escaped template
+==================
+
+foo = "$${ var.bar }"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier) 
+      (expression (expr_term (template_expr (quoted_template (escape_sequence))))))))

test/corpus/templates.txt

@@ -0,0 +1,13 @@
+==================
+quoted template expression
+==================
+
+foo = "${ var.bar }"
+
+---
+
+(config_file
+  (body
+    (attribute
+      (identifier)
+      (expression (expr_term (template_expr (quoted_template (template_interpolation))))))))