jenterkin/tree-sitter-jhcl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

handle quoted template expressions and template interpolations

Browse Source
This commit is contained in:
mhoffm
2021-06-14 00:55:24 +02:00
parent 3c984653bb
commit 7b91d51683
20 changed files with 5381 additions and 4751 deletions
Download Patch File Download Diff File Expand all files Collapse all files

197
src/grammar.json
View File

@@ -152,6 +152,10 @@
"type": "SYMBOL",
"name": "literal_value"
},
{
"type": "SYMBOL",
"name": "template_expr"
},
{
"type": "SYMBOL",
"name": "collection_value"
@@ -233,10 +237,6 @@
"type": "SYMBOL",
"name": "numeric_lit"
},
{
"type": "SYMBOL",
"name": "string_lit"
},
{
"type": "SYMBOL",
"name": "bool_lit"
@@ -251,87 +251,6 @@
"type": "PATTERN",
"value": "[0-9]+(\\.[0-9]+([eE][-+]?[0-9]+)?)?"
},
"string_lit": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PREC",
"value": 1,
"content": {
"type": "PATTERN",
"value": "[^\\\\\"\\n\\r\\t]+"
}
}
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
"type": "STRING",
"value": "\""
}
]
},
"escape_sequence": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\\"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "\\"
},
{
"type": "STRING",
"value": "\""
},
{
"type": "STRING",
"value": "n"
},
{
"type": "STRING",
"value": "r"
},
{
"type": "STRING",
"value": "t"
},
{
"type": "PATTERN",
"value": "u[0-9a-fA-F]{4}"
},
{
"type": "PATTERN",
"value": "U[0-9a-fA-F]{8}"
}
]
}
]
}
},
"bool_lit": {
"type": "CHOICE",
"members": [
@@ -1086,6 +1005,99 @@
}
]
},
"template_expr": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "quoted_template"
}
]
},
"string_lit": {
"type": "SYMBOL",
"name": "quoted_template"
},
"quoted_template": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_template_char"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
},
{
"type": "SYMBOL",
"name": "template_interpolation"
}
]
}
},
{
"type": "STRING",
"value": "\""
}
]
},
"template_interpolation": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "${"
},
{
"type": "STRING",
"value": "${~"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_template_char_in_interpolation"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "}"
},
{
"type": "STRING",
"value": "~}"
}
]
}
]
},
"comment": {
"type": "TOKEN",
"content": {
@@ -1167,7 +1179,20 @@
]
],
"precedences": [],
"externals": [],
"externals": [
{
"type": "SYMBOL",
"name": "_template_char"
},
{
"type": "SYMBOL",
"name": "_template_char_in_interpolation"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
],
"inline": [],
"supertypes": []
}

69
src/node-types.json
View File

@@ -206,6 +206,10 @@
"type": "splat",
"named": true
},
{
"type": "template_expr",
"named": true
},
{
"type": "variable_expr",
"named": true
@@ -452,10 +456,6 @@
{
"type": "numeric_lit",
"named": true
},
{
"type": "string_lit",
"named": true
}
]
}
@@ -513,6 +513,25 @@
]
}
},
{
"type": "quoted_template",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": false,
"types": [
{
"type": "escape_sequence",
"named": true
},
{
"type": "template_interpolation",
"named": true
}
]
}
},
{
"type": "splat",
"named": true,
@@ -536,6 +555,36 @@
"type": "string_lit",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
{
"type": "quoted_template",
"named": true
}
]
}
},
{
"type": "template_expr",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
{
"type": "quoted_template",
"named": true
}
]
}
},
{
"type": "template_interpolation",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": false,
@@ -604,6 +653,14 @@
"type": "\"",
"named": false
},
{
"type": "${",
"named": false
},
{
"type": "${~",
"named": false
},
{
"type": "%",
"named": false
@@ -743,5 +800,9 @@
{
"type": "}",
"named": false
},
{
"type": "~}",
"named": false
}
]

9046
src/parser.c
View File

File diff suppressed because it is too large Load Diff

148
src/scanner.c Normal file
View File

@@ -0,0 +1,148 @@
#include <tree_sitter/parser.h>
#include <wctype.h>
// forward declarations
enum TokenType {
TEMPLATE_CHAR,
TEMPLATE_CHAR_IN_INTERPOLATION,
ESCAPE_SEQUENCE
};
// helpers
static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static bool accept_template_char_inplace(TSLexer *lexer) {
lexer->result_symbol = TEMPLATE_CHAR;
return true;
}
static bool advance_and_accept_template_char(TSLexer *lexer) {
advance(lexer);
return accept_template_char_inplace(lexer);
}
static bool advance_and_accept_escape_sequence(TSLexer *lexer) {
lexer->result_symbol = ESCAPE_SEQUENCE;
advance(lexer);
return true;
}
static bool advance_and_accept_template_char_in_interpolation(TSLexer *lexer) {
advance(lexer);
lexer->result_symbol = TEMPLATE_CHAR_IN_INTERPOLATION;
return true;
}
static bool consume_wxdigit(TSLexer *lexer) {
advance(lexer);
return iswxdigit(lexer->lookahead);
}
// scan escape sequences \n \t \r \\ \" \uHHHH or \UHHHHHHHHA where H is a hex digit
// assumes that the leading character is '\'
static bool scan_backslash_escape_sequence(TSLexer *lexer) {
advance(lexer);
switch (lexer->lookahead) {
case '"':
case 'n':
case 'r':
case 't':
case '\\':
return advance_and_accept_escape_sequence(lexer);
case 'u':
for (int i = 0; i < 4; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return advance_and_accept_escape_sequence(lexer);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return advance_and_accept_escape_sequence(lexer);
default:
return false;
}
}
// may accept multiple characters like %% as a 'template literal chunk'
// assumes that the leading character is '%' or '$'
static bool scan_template_literal_chunk_or_template_escape_sequence(TSLexer *lexer) {
const leading_char = lexer->lookahead;
advance(lexer);
// reject %{ because its the start of template directives
if (lexer->lookahead == '{') return false;
if (lexer->lookahead == leading_char) {
advance(lexer);
// accept %%{ as escape sequence
if (lexer->lookahead == '{') return advance_and_accept_escape_sequence(lexer);
}
// accept % and %% as template chars
return accept_template_char_inplace(lexer);
}
static bool scan_template_char_or_escape_sequence(TSLexer *lexer) {
switch (lexer->lookahead) {
case '"':
case '\n':
case '\r':
case '\t':
return false;
case '\\':
return scan_backslash_escape_sequence(lexer);
case '$':
case '%':
return scan_template_literal_chunk_or_template_escape_sequence(lexer);
default:
return advance_and_accept_template_char(lexer);
}
}
static bool scan_template_char_in_interpolation_or_escape_sequence(TSLexer *lexer) {
switch (lexer->lookahead) {
case '"':
case '\n':
case '\r':
case '\t':
// no template interpolation chars are allowed in template interpolations ( even escaped )
case '$':
case '%':
case '~':
return false;
// '}' ends the template interpolation
case '}':
return false;
case '\\':
return scan_backslash_escape_sequence(lexer);
default:
return advance_and_accept_template_char_in_interpolation(lexer);
}
}
// scanner
void *tree_sitter_hcl_external_scanner_create() { return NULL; }
void tree_sitter_hcl_external_scanner_destroy(void *p) {}
void tree_sitter_hcl_external_scanner_reset(void *p) {}
unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) { return 0; }
void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
bool tree_sitter_hcl_external_scanner_scan(
void *p,
TSLexer *lexer,
const bool *valid_symbols
) {
// when scanning string literals or quoted template literals that are not in an template directive
if (valid_symbols[TEMPLATE_CHAR] && valid_symbols[ESCAPE_SEQUENCE]) {
return scan_template_char_or_escape_sequence(lexer);
}
// quoted template literals currently inside a template directive
if (valid_symbols[TEMPLATE_CHAR_IN_INTERPOLATION] && valid_symbols[ESCAPE_SEQUENCE]) {
return scan_template_char_in_interpolation_or_escape_sequence(lexer);
}
return false;
}