jenterkin/tree-sitter-jhcl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify whitespace handling; support hexadecimal numeric literals; allow empty configs

Browse Source
This commit is contained in:
mhoffm
2021-06-20 00:03:58 +02:00
parent b1c3109853
commit 22d0a7253c
15 changed files with 12984 additions and 13226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
example/example.hcl
View File

@@ -36,5 +36,6 @@ resource_1 "strlit1" "strlit2" {
attr4 = 2.112e+12 attr4 = 2.112e+12
attr5 = 2.112E+12 attr5 = 2.112E+12
attr6 = 2.112E-12 attr6 = 2.112E-12
attr7 = 0x21FF
} }
} }

BIN
example/real_world_stuff/hashicorp%field-demos-consul-expand-team-qapla-assetsconsul-configs%frontend-router.hcl
View File

Binary file not shown.

BIN
example/real_world_stuff/hashicorp%field-demos-consul-expand-team-qapla-assetsconsul-configs%proxydefaults.hcl
View File

Binary file not shown.

BIN
example/real_world_stuff/hashicorp%field-demos-consul-expand-team-qapla-assetsmodules%tf-gke%providers.tf
View File

Binary file not shown.

BIN
example/real_world_stuff/hashicorp%terraforminternal%initwd%testdata%load-module-package-escape%grandchild%package-escape-grandchild.tf
View File

Binary file not shown.

BIN
example/real_world_stuff/hashicorp%terraforminternal%initwd%testdata%load-module-package-prefix%package%grandchild%package-prefix-grandchild.tf
View File

Binary file not shown.

81
grammar.js
View File

@@ -15,12 +15,9 @@ module.exports = grammar({
conflicts: $ => [ conflicts: $ => [
// string literals are just quoted template without template stuff // string literals are just quoted template without template stuff
[$.string_lit, $.quoted_template], [$.string_lit, $.quoted_template],
// empty block may be both
[$.block, $.one_line_block],
], ],
externals: $ => [ externals: $ => [
$._newline,
$._quoted_template_start, $._quoted_template_start,
$._quoted_template_end, $._quoted_template_end,
$._template_literal_chunk, $._template_literal_chunk,
@@ -30,46 +27,31 @@ module.exports = grammar({
extras: $ => [ extras: $ => [
$.comment, $.comment,
' ', $._whitespace,
'\t',
], ],
rules: { rules: {
config_file: $ => $.body, config_file: $ => optional($.body),
body: $ => repeat1(prec.left( body: $ => repeat1(
choice( choice(
$.attribute, $.attribute,
$.block, $.block,
$.one_line_block,
$._newlines,
), ),
)), ),
attribute: $ => prec.left(seq( attribute: $ => seq(
$.identifier, $.identifier,
'=', '=',
$.expression, $.expression,
$._newlines, ),
)),
block: $ => prec.left(seq( block: $ => seq(
$.identifier, $.identifier,
repeat(choice($.string_lit, $.identifier)), repeat(choice($.string_lit, $.identifier)),
$._block_start, $._block_start,
optional($._newlines),
optional($.body), optional($.body),
$._block_end, $._block_end,
optional($._newlines),
)),
one_line_block: $ => seq(
$.identifier,
repeat(choice($.string_lit, $.identifier)),
$._block_start,
optional(seq($.identifier, '=', $.expression)),
$._block_end,
$._newlines,
), ),
_block_start: $ => '{', _block_start: $ => '{',
@@ -77,14 +59,14 @@ module.exports = grammar({
// TODO: not to spec but good enough for now // TODO: not to spec but good enough for now
identifier: $ => token(seq( identifier: $ => token(seq(
/\p{L}/, choice(/\p{L}/, '_'),
repeat(choice(/\p{L}/, /[0-9]/, /(-|_)/)), repeat(choice(/\p{L}/, /[0-9]/, /(-|_)/)),
)), )),
expression: $ => choice( expression: $ => prec.right(choice(
$._expr_term, $._expr_term,
$.conditional, $.conditional,
), )),
// operations are documented as expressions, but our real world samples // operations are documented as expressions, but our real world samples
// contain instances of operations without parentheses. think for example: // contain instances of operations without parentheses. think for example:
@@ -110,7 +92,10 @@ module.exports = grammar({
$.string_lit, $.string_lit,
), ),
numeric_lit: $ => /[0-9]+(\.[0-9]+([eE][-+]?[0-9]+)?)?/, numeric_lit: $ => choice(
/[0-9]+(\.[0-9]+([eE][-+]?[0-9]+)?)?/,
/0x[0-9a-zA-Z]+/
),
bool_lit: $ => choice('true', 'false'), bool_lit: $ => choice('true', 'false'),
@@ -133,41 +118,38 @@ module.exports = grammar({
tuple: $ => seq( tuple: $ => seq(
$._tuple_start, $._tuple_start,
optional($._tuple_elems), optional($._tuple_elems),
optional($._newlines),
$._tuple_end, $._tuple_end,
), ),
_tuple_start: $ => '[', _tuple_start: $ => '[',
_tuple_end: $ => ']', _tuple_end: $ => ']',
_tuple_elems: $ => prec.right(seq( _tuple_elems: $ => seq(
optional($._newlines),
$.expression, $.expression,
repeat(seq($._comma, optional($._newlines), $.expression)), repeat(seq(
$._comma,
$.expression,
)),
optional($._comma), optional($._comma),
optional($._newlines), ),
)),
object: $ => seq( object: $ => seq(
$._object_start, $._object_start,
optional($._object_elems), optional($._object_elems),
optional($._newlines),
$._object_end, $._object_end,
), ),
_object_start: $ => '{', _object_start: $ => '{',
_object_end: $ => '}', _object_end: $ => '}',
_object_elems: $ => prec.right(seq( _object_elems: $ => seq(
optional($._newlines),
$.object_elem, $.object_elem,
repeat(seq( repeat(seq(
choice($._comma, $._newlines, seq($._comma, $._newlines)), optional($._comma),
$.object_elem $.object_elem
)), )),
optional($._comma), optional($._comma),
optional($._newlines), ),
)),
object_elem: $ => seq( object_elem: $ => seq(
$.expression, $.expression,
@@ -196,28 +178,22 @@ module.exports = grammar({
for_expr: $ => choice($.for_tuple_expr, $.for_object_expr), for_expr: $ => choice($.for_tuple_expr, $.for_object_expr),
// newlines
for_tuple_expr: $ => seq( for_tuple_expr: $ => seq(
$._tuple_start, $._tuple_start,
optional($._newlines),
$.for_intro, $.for_intro,
$.expression, $.expression,
optional($.for_cond), optional($.for_cond),
optional($._newlines),
$._tuple_end, $._tuple_end,
), ),
// newlines
for_object_expr: $ => seq( for_object_expr: $ => seq(
$._object_start, $._object_start,
optional($._newlines),
$.for_intro, $.for_intro,
$.expression, $.expression,
'=>', '=>',
$.expression, $.expression,
optional($.ellipsis), optional($.ellipsis),
optional($.for_cond), optional($.for_cond),
optional($._newlines),
$._object_end, $._object_end,
), ),
@@ -235,13 +211,12 @@ module.exports = grammar({
$.expression, $.expression,
), ),
variable_expr: $ => $.identifier, variable_expr: $ => prec.right($.identifier),
function_call: $ => seq( function_call: $ => seq(
$.identifier, $.identifier,
$._function_call_start, $._function_call_start,
optional($.function_arguments), optional($.function_arguments),
optional($._newlines),
$._function_call_end, $._function_call_end,
), ),
@@ -249,10 +224,8 @@ module.exports = grammar({
_function_call_end: $ => ')', _function_call_end: $ => ')',
function_arguments: $ => prec.right(seq( function_arguments: $ => prec.right(seq(
optional($._newline),
$.expression, $.expression,
repeat(seq(',', optional($._newline), $.expression,)), repeat(seq($._comma, $.expression,)),
optional($._newline),
optional(choice(',', $.ellipsis)), optional(choice(',', $.ellipsis)),
)), )),
@@ -321,8 +294,6 @@ module.exports = grammar({
//$.template_if, //$.template_if,
), ),
_newlines: $ => prec.right(repeat1($._newline)),
// http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890 // http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890
comment: $ => token(choice( comment: $ => token(choice(
seq('#', /.*/), seq('#', /.*/),
@@ -333,5 +304,7 @@ module.exports = grammar({
'/' '/'
) )
)), )),
_whitespace: $ => token(/\s/),
} }
}); });

598
src/grammar.json
View File

@@ -2,136 +2,51 @@
"name": "hcl", "name": "hcl",
"rules": { "rules": {
"config_file": { "config_file": {
"type": "SYMBOL", "type": "CHOICE",
"name": "body" "members": [
{
"type": "SYMBOL",
"name": "body"
},
{
"type": "BLANK"
}
]
}, },
"body": { "body": {
"type": "REPEAT1", "type": "REPEAT1",
"content": { "content": {
"type": "PREC_LEFT", "type": "CHOICE",
"value": 0, "members": [
"content": { {
"type": "CHOICE", "type": "SYMBOL",
"members": [ "name": "attribute"
{ },
"type": "SYMBOL", {
"name": "attribute" "type": "SYMBOL",
}, "name": "block"
{ }
"type": "SYMBOL", ]
"name": "block"
},
{
"type": "SYMBOL",
"name": "one_line_block"
},
{
"type": "SYMBOL",
"name": "_newlines"
}
]
}
} }
}, },
"attribute": { "attribute": {
"type": "PREC_LEFT", "type": "SEQ",
"value": 0, "members": [
"content": { {
"type": "SEQ", "type": "SYMBOL",
"members": [ "name": "identifier"
{ },
"type": "SYMBOL", {
"name": "identifier" "type": "STRING",
}, "value": "="
{ },
"type": "STRING", {
"value": "=" "type": "SYMBOL",
}, "name": "expression"
{ }
"type": "SYMBOL", ]
"name": "expression"
},
{
"type": "SYMBOL",
"name": "_newlines"
}
]
}
}, },
"block": { "block": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "string_lit"
},
{
"type": "SYMBOL",
"name": "identifier"
}
]
}
},
{
"type": "SYMBOL",
"name": "_block_start"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "body"
},
{
"type": "BLANK"
}
]
},
{
"type": "SYMBOL",
"name": "_block_end"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
}
]
}
},
"one_line_block": {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
@@ -162,21 +77,8 @@
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
{ {
"type": "SEQ", "type": "SYMBOL",
"members": [ "name": "body"
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "expression"
}
]
}, },
{ {
"type": "BLANK" "type": "BLANK"
@@ -186,10 +88,6 @@
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_block_end" "name": "_block_end"
},
{
"type": "SYMBOL",
"name": "_newlines"
} }
] ]
}, },
@@ -207,8 +105,17 @@
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
"type": "PATTERN", "type": "CHOICE",
"value": "\\p{L}" "members": [
{
"type": "PATTERN",
"value": "\\p{L}"
},
{
"type": "STRING",
"value": "_"
}
]
}, },
{ {
"type": "REPEAT", "type": "REPEAT",
@@ -234,17 +141,21 @@
} }
}, },
"expression": { "expression": {
"type": "CHOICE", "type": "PREC_RIGHT",
"members": [ "value": 0,
{ "content": {
"type": "SYMBOL", "type": "CHOICE",
"name": "_expr_term" "members": [
}, {
{ "type": "SYMBOL",
"type": "SYMBOL", "name": "_expr_term"
"name": "conditional" },
} {
] "type": "SYMBOL",
"name": "conditional"
}
]
}
}, },
"_expr_term": { "_expr_term": {
"type": "CHOICE", "type": "CHOICE",
@@ -357,8 +268,17 @@
] ]
}, },
"numeric_lit": { "numeric_lit": {
"type": "PATTERN", "type": "CHOICE",
"value": "[0-9]+(\\.[0-9]+([eE][-+]?[0-9]+)?)?" "members": [
{
"type": "PATTERN",
"value": "[0-9]+(\\.[0-9]+([eE][-+]?[0-9]+)?)?"
},
{
"type": "PATTERN",
"value": "0x[0-9a-zA-Z]+"
}
]
}, },
"bool_lit": { "bool_lit": {
"type": "CHOICE", "type": "CHOICE",
@@ -430,18 +350,6 @@
} }
] ]
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_tuple_end" "name": "_tuple_end"
@@ -457,81 +365,41 @@
"value": "]" "value": "]"
}, },
"_tuple_elems": { "_tuple_elems": {
"type": "PREC_RIGHT", "type": "SEQ",
"value": 0, "members": [
"content": { {
"type": "SEQ", "type": "SYMBOL",
"members": [ "name": "expression"
{ },
"type": "CHOICE", {
"members": [ "type": "REPEAT",
{ "content": {
"type": "SYMBOL", "type": "SEQ",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{
"type": "SYMBOL",
"name": "expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_comma"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{
"type": "SYMBOL",
"name": "expression"
}
]
}
},
{
"type": "CHOICE",
"members": [ "members": [
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_comma" "name": "_comma"
}, },
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_newlines" "name": "expression"
},
{
"type": "BLANK"
} }
] ]
} }
] },
} {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_comma"
},
{
"type": "BLANK"
}
]
}
]
}, },
"object": { "object": {
"type": "SEQ", "type": "SEQ",
@@ -552,18 +420,6 @@
} }
] ]
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_object_end" "name": "_object_end"
@@ -579,91 +435,49 @@
"value": "}" "value": "}"
}, },
"_object_elems": { "_object_elems": {
"type": "PREC_RIGHT", "type": "SEQ",
"value": 0, "members": [
"content": { {
"type": "SEQ", "type": "SYMBOL",
"members": [ "name": "object_elem"
{ },
"type": "CHOICE", {
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [ "members": [
{ {
"type": "SYMBOL", "type": "CHOICE",
"name": "_newlines" "members": [
{
"type": "SYMBOL",
"name": "_comma"
},
{
"type": "BLANK"
}
]
}, },
{
"type": "BLANK"
}
]
},
{
"type": "SYMBOL",
"name": "object_elem"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_comma"
},
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_comma"
},
{
"type": "SYMBOL",
"name": "_newlines"
}
]
}
]
},
{
"type": "SYMBOL",
"name": "object_elem"
}
]
}
},
{
"type": "CHOICE",
"members": [
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_comma" "name": "object_elem"
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
} }
] ]
} }
] },
} {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_comma"
},
{
"type": "BLANK"
}
]
}
]
}, },
"object_elem": { "object_elem": {
"type": "SEQ", "type": "SEQ",
@@ -829,18 +643,6 @@
"type": "SYMBOL", "type": "SYMBOL",
"name": "_tuple_start" "name": "_tuple_start"
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "for_intro" "name": "for_intro"
@@ -861,18 +663,6 @@
} }
] ]
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_tuple_end" "name": "_tuple_end"
@@ -886,18 +676,6 @@
"type": "SYMBOL", "type": "SYMBOL",
"name": "_object_start" "name": "_object_start"
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "for_intro" "name": "for_intro"
@@ -938,18 +716,6 @@
} }
] ]
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_object_end" "name": "_object_end"
@@ -1016,8 +782,12 @@
] ]
}, },
"variable_expr": { "variable_expr": {
"type": "SYMBOL", "type": "PREC_RIGHT",
"name": "identifier" "value": 0,
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}, },
"function_call": { "function_call": {
"type": "SEQ", "type": "SEQ",
@@ -1042,18 +812,6 @@
} }
] ]
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newlines"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_function_call_end" "name": "_function_call_end"
@@ -1074,18 +832,6 @@
"content": { "content": {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newline"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "expression" "name": "expression"
@@ -1096,20 +842,8 @@
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
"type": "STRING", "type": "SYMBOL",
"value": "," "name": "_comma"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newline"
},
{
"type": "BLANK"
}
]
}, },
{ {
"type": "SYMBOL", "type": "SYMBOL",
@@ -1118,18 +852,6 @@
] ]
} }
}, },
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newline"
},
{
"type": "BLANK"
}
]
},
{ {
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
@@ -1520,17 +1242,6 @@
"type": "CHOICE", "type": "CHOICE",
"members": [] "members": []
}, },
"_newlines": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "_newline"
}
}
},
"comment": { "comment": {
"type": "TOKEN", "type": "TOKEN",
"content": { "content": {
@@ -1581,6 +1292,13 @@
} }
] ]
} }
},
"_whitespace": {
"type": "TOKEN",
"content": {
"type": "PATTERN",
"value": "\\s"
}
} }
}, },
"extras": [ "extras": [
@@ -1589,30 +1307,18 @@
"name": "comment" "name": "comment"
}, },
{ {
"type": "STRING", "type": "SYMBOL",
"value": " " "name": "_whitespace"
},
{
"type": "STRING",
"value": "\t"
} }
], ],
"conflicts": [ "conflicts": [
[ [
"string_lit", "string_lit",
"quoted_template" "quoted_template"
],
[
"block",
"one_line_block"
] ]
], ],
"precedences": [], "precedences": [],
"externals": [ "externals": [
{
"type": "SYMBOL",
"name": "_newline"
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_quoted_template_start" "name": "_quoted_template_start"

40
src/node-types.json
View File

@@ -117,7 +117,7 @@
"fields": {}, "fields": {},
"children": { "children": {
"multiple": true, "multiple": true,
"required": false, "required": true,
"types": [ "types": [
{ {
"type": "attribute", "type": "attribute",
@@ -126,10 +126,6 @@
{ {
"type": "block", "type": "block",
"named": true "named": true
},
{
"type": "one_line_block",
"named": true
} }
] ]
} }
@@ -179,7 +175,7 @@
"fields": {}, "fields": {},
"children": { "children": {
"multiple": false, "multiple": false,
"required": true, "required": false,
"types": [ "types": [
{ {
"type": "body", "type": "body",
@@ -488,6 +484,11 @@
] ]
} }
}, },
{
"type": "numeric_lit",
"named": true,
"fields": {}
},
{ {
"type": "object", "type": "object",
"named": true, "named": true,
@@ -518,29 +519,6 @@
] ]
} }
}, },
{
"type": "one_line_block",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "identifier",
"named": true
},
{
"type": "string_lit",
"named": true
}
]
}
},
{ {
"type": "operation", "type": "operation",
"named": true, "named": true,
@@ -878,10 +856,6 @@
"type": "null_lit", "type": "null_lit",
"named": true "named": true
}, },
{
"type": "numeric_lit",
"named": true
},
{ {
"type": "strip_marker", "type": "strip_marker",
"named": true "named": true

25382
src/parser.c
View File

File diff suppressed because it is too large Load Diff

25
src/scanner.c
View File

@@ -5,7 +5,6 @@
#include <stdio.h> #include <stdio.h>
enum TokenType { enum TokenType {
NEWLINE,
QUOTED_TEMPLATE_START, QUOTED_TEMPLATE_START,
QUOTED_TEMPLATE_END, QUOTED_TEMPLATE_END,
TEMPLATE_LITERAL_CHUNK, TEMPLATE_LITERAL_CHUNK,
@@ -48,7 +47,6 @@ void print_debug_info(Scanner *scanner, TSLexer *lexer, const bool *valid_symbol
printf("template_literal_chunk: %x\n", valid_symbols[TEMPLATE_LITERAL_CHUNK]); printf("template_literal_chunk: %x\n", valid_symbols[TEMPLATE_LITERAL_CHUNK]);
printf("template_interpolation_start: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_START]); printf("template_interpolation_start: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_START]);
printf("template_interpolation_end: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_END]); printf("template_interpolation_end: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_END]);
printf("newline: %x\n", valid_symbols[NEWLINE]);
printf("\n"); printf("\n");
printf("scanner state:\n"); printf("scanner state:\n");
printf("in_template_interpolation %x\n", scanner->in_template_interpolation); printf("in_template_interpolation %x\n", scanner->in_template_interpolation);
@@ -86,31 +84,13 @@ void scanner_exit_quoted_context(Scanner *scanner) {
} }
} }
bool is_newline(char c) {
return c == '\n' || c == '\r';
}
bool is_skippable_whitespace_outside_of_quoted_context(char c) {
return c == ' ' || c == '\t';
}
bool scanner_scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) { bool scanner_scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
// print_debug_info(scanner, lexer, valid_symbols); // print_debug_info(scanner, lexer, valid_symbols);
while ( while (iswspace(lexer->lookahead) && !scanner->in_quoted_context) {
is_skippable_whitespace_outside_of_quoted_context(lexer->lookahead) &&
!scanner->in_quoted_context
) {
skip(lexer); skip(lexer);
} }
if (valid_symbols[NEWLINE] &&
is_newline(lexer->lookahead) &&
scanner->quoted_context_depth == 0
) {
return accept_and_advance(lexer, NEWLINE);
}
// manage quoted context // manage quoted context
if ( if (
valid_symbols[QUOTED_TEMPLATE_START] && valid_symbols[QUOTED_TEMPLATE_START] &&
@@ -162,9 +142,6 @@ bool scanner_scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
// //
// they may not contain newlines and may contain escape sequences // they may not contain newlines and may contain escape sequences
if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && scanner->in_quoted_context) { if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && scanner->in_quoted_context) {
if (is_newline(lexer->lookahead)) {
return false;
}
switch (lexer->lookahead) { switch (lexer->lookahead) {
case '\\': case '\\':
advance(lexer); advance(lexer);

38
test/corpus/blocks.txt
View File

@@ -10,8 +10,7 @@ block_1 {
(config_file (config_file
(body (body
(block (block
(identifier) (identifier))))
(body))))
================================================================================ ================================================================================
basic block on one line basic block on one line
@@ -69,8 +68,7 @@ block_1 {
(identifier) (identifier)
(body (body
(block (block
(identifier) (identifier))))))
(body))))))
================================================================================ ================================================================================
nested block on one line nested block on one line
@@ -122,20 +120,22 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
(config_file (config_file
(body (body
(one_line_block (block
(identifier) (identifier)
(identifier) (body
(expression (attribute
(function_call
(identifier) (identifier)
(function_arguments (expression
(expression (function_call
(function_call (identifier)
(identifier))) (function_arguments
(expression (expression
(literal_value (function_call
(string_lit (identifier)))
(template_literal)))) (expression
(expression (literal_value
(template_expr (string_lit
(quoted_template))))))))) (template_literal))))
(expression
(template_expr
(quoted_template)))))))))))

1
test/corpus/collections.txt
View File

@@ -365,3 +365,4 @@ worker_groups = [
(expression (expression
(literal_value (literal_value
(numeric_lit))))))))))))) (numeric_lit)))))))))))))

16
test/corpus/literals.txt
View File

@@ -1,3 +1,19 @@
================================================================================
numeric literal hex 1
================================================================================
pi = 0x314F
--------------------------------------------------------------------------------
(config_file
(body
(attribute
(identifier)
(expression
(literal_value
(numeric_lit))))))
================================================================================ ================================================================================
numeric literal scientific notation 1 numeric literal scientific notation 1
================================================================================ ================================================================================

28
test/corpus/strings.txt
View File

@@ -11,32 +11,12 @@ foo = "bar\uZZ"
(attribute (attribute
(identifier) (identifier)
(expression (expression
(literal_value (template_expr
(string_lit (quoted_template
(template_literal (template_literal
(ERROR (ERROR
(UNEXPECTED '\'))))))))) (UNEXPECTED '\')))))))))
================================================================================
literal multi line error
================================================================================
foo = "
bar"
--------------------------------------------------------------------------------
(config_file
(body
(attribute
(identifier)
(expression
(literal_value
(string_lit
(ERROR
(UNEXPECTED '\n'))
(template_literal)))))))
================================================================================ ================================================================================
unescaped tab unescaped tab
================================================================================ ================================================================================
@@ -67,8 +47,8 @@ foo = "foo\bar"
(attribute (attribute
(identifier) (identifier)
(expression (expression
(literal_value (template_expr
(string_lit (quoted_template
(template_literal (template_literal
(ERROR (ERROR
(UNEXPECTED '\'))))))))) (UNEXPECTED '\')))))))))