jenterkin/tree-sitter-jhcl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fmt tests; properly define string literals; work on scanner

Browse Source
This commit is contained in:
mhoffm
2021-06-17 19:39:46 +02:00
parent 4801d16825
commit 88401e3bf3
20 changed files with 8036 additions and 5094 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
src/grammar.json
View File

@@ -75,8 +75,8 @@
}
},
{
"type": "STRING",
"value": "{"
"type": "SYMBOL",
"name": "_block_start"
},
{
"type": "CHOICE",
@@ -91,11 +91,19 @@
]
},
{
"type": "STRING",
"value": "}"
"type": "SYMBOL",
"name": "_block_end"
}
]
},
"_block_start": {
"type": "STRING",
"value": "{"
},
"_block_end": {
"type": "STRING",
"value": "}"
},
"identifier": {
"type": "TOKEN",
"content": {
@@ -133,7 +141,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "SYMBOL",
@@ -145,7 +153,7 @@
}
]
},
"expr_term": {
"_expr_term": {
"type": "CHOICE",
"members": [
{
@@ -177,7 +185,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "SYMBOL",
@@ -190,7 +198,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "SYMBOL",
@@ -203,7 +211,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "SYMBOL",
@@ -244,6 +252,10 @@
{
"type": "SYMBOL",
"name": "null_lit"
},
{
"type": "SYMBOL",
"name": "string_lit"
}
]
},
@@ -268,6 +280,23 @@
"type": "STRING",
"value": "null"
},
"string_lit": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_quoted_template_start"
},
{
"type": "SYMBOL",
"name": "template_literal"
},
{
"type": "SYMBOL",
"name": "_quoted_template_end"
}
]
},
"collection_value": {
"type": "CHOICE",
"members": [
@@ -331,8 +360,8 @@
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "{"
"type": "SYMBOL",
"name": "_object_start"
},
{
"type": "CHOICE",
@@ -368,11 +397,19 @@
]
},
{
"type": "STRING",
"value": "}"
"type": "SYMBOL",
"name": "_object_end"
}
]
},
"_object_start": {
"type": "STRING",
"value": "{"
},
"_object_end": {
"type": "STRING",
"value": "}"
},
"object_elem": {
"type": "SEQ",
"members": [
@@ -554,8 +591,8 @@
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "{"
"type": "SYMBOL",
"name": "_object_start"
},
{
"type": "SYMBOL",
@@ -598,8 +635,8 @@
]
},
{
"type": "STRING",
"value": "}"
"type": "SYMBOL",
"name": "_object_end"
}
]
},
@@ -678,8 +715,8 @@
}
},
{
"type": "STRING",
"value": "("
"type": "SYMBOL",
"name": "_function_call_start"
},
{
"type": "CHOICE",
@@ -694,11 +731,19 @@
]
},
{
"type": "STRING",
"value": ")"
"type": "SYMBOL",
"name": "_function_call_end"
}
]
},
"_function_call_start": {
"type": "STRING",
"value": "("
},
"_function_call_end": {
"type": "STRING",
"value": ")"
},
"function_arguments": {
"type": "SEQ",
"members": [
@@ -811,7 +856,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -827,7 +872,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "CHOICE",
@@ -848,7 +893,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -861,7 +906,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "CHOICE",
@@ -878,7 +923,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -891,7 +936,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "CHOICE",
@@ -916,7 +961,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -929,7 +974,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "CHOICE",
@@ -946,7 +991,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -959,7 +1004,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "CHOICE",
@@ -972,7 +1017,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -985,7 +1030,7 @@
"members": [
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
},
{
"type": "CHOICE",
@@ -998,7 +1043,7 @@
},
{
"type": "SYMBOL",
"name": "expr_term"
"name": "_expr_term"
}
]
}
@@ -1014,41 +1059,12 @@
}
]
},
"string_lit": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_template_char"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
"type": "STRING",
"value": "\""
}
]
},
"quoted_template": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
"type": "SYMBOL",
"name": "_quoted_template_start"
},
{
"type": "REPEAT",
@@ -1057,72 +1073,85 @@
"members": [
{
"type": "SYMBOL",
"name": "_template_char"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
"name": "template_literal"
},
{
"type": "SYMBOL",
"name": "template_interpolation"
},
{
"type": "SYMBOL",
"name": "template_directive"
}
]
}
},
{
"type": "STRING",
"value": "\""
"type": "SYMBOL",
"name": "_quoted_template_end"
}
]
},
"strip_marker": {
"type": "STRING",
"value": "~"
},
"template_literal": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "_template_literal_chunk"
}
}
},
"template_interpolation": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "${"
},
{
"type": "STRING",
"value": "${~"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_template_char_in_interpolation"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
"type": "SYMBOL",
"name": "_template_interpolation_start"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "}"
"type": "SYMBOL",
"name": "strip_marker"
},
{
"type": "STRING",
"value": "~}"
"type": "BLANK"
}
]
},
{
"type": "SYMBOL",
"name": "expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "strip_marker"
},
{
"type": "BLANK"
}
]
},
{
"type": "SYMBOL",
"name": "_template_interpolation_end"
}
]
},
"template_directive": {
"type": "CHOICE",
"members": []
},
"comment": {
"type": "TOKEN",
"content": {
@@ -1211,15 +1240,23 @@
"externals": [
{
"type": "SYMBOL",
"name": "_template_char"
"name": "_quoted_template_start"
},
{
"type": "SYMBOL",
"name": "_template_char_in_interpolation"
"name": "_quoted_template_end"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
"name": "_template_literal_chunk"
},
{
"type": "SYMBOL",
"name": "_template_interpolation_start"
},
{
"type": "SYMBOL",
"name": "_template_interpolation_end"
}
],
"inline": [],

167
src/node-types.json
View File

@@ -49,7 +49,43 @@
"required": true,
"types": [
{
"type": "expr_term",
"type": "collection_value",
"named": true
},
{
"type": "expression",
"named": true
},
{
"type": "for_expr",
"named": true
},
{
"type": "function_call",
"named": true
},
{
"type": "get_attr",
"named": true
},
{
"type": "index",
"named": true
},
{
"type": "literal_value",
"named": true
},
{
"type": "splat",
"named": true
},
{
"type": "template_expr",
"named": true
},
{
"type": "variable_expr",
"named": true
}
]
@@ -163,7 +199,7 @@
}
},
{
"type": "expr_term",
"type": "expression",
"named": true,
"fields": {},
"children": {
@@ -175,7 +211,7 @@
"named": true
},
{
"type": "expr_term",
"type": "conditional",
"named": true
},
{
@@ -202,6 +238,10 @@
"type": "literal_value",
"named": true
},
{
"type": "operation",
"named": true
},
{
"type": "splat",
"named": true
@@ -217,29 +257,6 @@
]
}
},
{
"type": "expression",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
{
"type": "conditional",
"named": true
},
{
"type": "expr_term",
"named": true
},
{
"type": "operation",
"named": true
}
]
}
},
{
"type": "for_cond",
"named": true,
@@ -456,6 +473,10 @@
{
"type": "numeric_lit",
"named": true
},
{
"type": "string_lit",
"named": true
}
]
}
@@ -522,12 +543,16 @@
"required": false,
"types": [
{
"type": "escape_sequence",
"type": "template_directive",
"named": true
},
{
"type": "template_interpolation",
"named": true
},
{
"type": "template_literal",
"named": true
}
]
}
@@ -556,16 +581,21 @@
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": false,
"multiple": false,
"required": true,
"types": [
{
"type": "escape_sequence",
"type": "template_literal",
"named": true
}
]
}
},
{
"type": "template_directive",
"named": true,
"fields": {}
},
{
"type": "template_expr",
"named": true,
@@ -587,15 +617,24 @@
"fields": {},
"children": {
"multiple": true,
"required": false,
"required": true,
"types": [
{
"type": "escape_sequence",
"type": "expression",
"named": true
},
{
"type": "strip_marker",
"named": true
}
]
}
},
{
"type": "template_literal",
"named": true,
"fields": {}
},
{
"type": "tuple",
"named": true,
@@ -616,11 +655,47 @@
"named": true,
"fields": {},
"children": {
"multiple": false,
"multiple": true,
"required": true,
"types": [
{
"type": "expr_term",
"type": "collection_value",
"named": true
},
{
"type": "expression",
"named": true
},
{
"type": "for_expr",
"named": true
},
{
"type": "function_call",
"named": true
},
{
"type": "get_attr",
"named": true
},
{
"type": "index",
"named": true
},
{
"type": "literal_value",
"named": true
},
{
"type": "splat",
"named": true
},
{
"type": "template_expr",
"named": true
},
{
"type": "variable_expr",
"named": true
}
]
@@ -649,18 +724,6 @@
"type": "!=",
"named": false
},
{
"type": "\"",
"named": false
},
{
"type": "${",
"named": false
},
{
"type": "${~",
"named": false
},
{
"type": "%",
"named": false
@@ -753,10 +816,6 @@
"type": "ellipsis",
"named": true
},
{
"type": "escape_sequence",
"named": true
},
{
"type": "false",
"named": false
@@ -785,6 +844,10 @@
"type": "numeric_lit",
"named": true
},
{
"type": "strip_marker",
"named": true
},
{
"type": "true",
"named": false
@@ -800,9 +863,5 @@
{
"type": "}",
"named": false
},
{
"type": "~}",
"named": false
}
]

11254
src/parser.c
View File

File diff suppressed because it is too large Load Diff

284
src/scanner.c
View File

@@ -1,144 +1,204 @@
#include <tree_sitter/parser.h>
#include <wctype.h>
#include <string.h>
#include <stdio.h>
enum TokenType {
TEMPLATE_CHAR,
TEMPLATE_CHAR_IN_INTERPOLATION,
ESCAPE_SEQUENCE
QUOTED_TEMPLATE_START,
QUOTED_TEMPLATE_END,
TEMPLATE_LITERAL_CHUNK,
TEMPLATE_INTERPOLATION_START,
TEMPLATE_INTERPOLATION_END,
};
static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static bool accept_template_char_inplace(TSLexer *lexer) {
lexer->result_symbol = TEMPLATE_CHAR;
return true;
static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
static bool accept_inplace(TSLexer *lexer, enum TokenType token) {
lexer->result_symbol = token;
return true;
}
static bool advance_and_accept_template_char(TSLexer *lexer) {
advance(lexer);
return accept_template_char_inplace(lexer);
}
static bool advance_and_accept_escape_sequence(TSLexer *lexer) {
advance(lexer);
lexer->result_symbol = ESCAPE_SEQUENCE;
return true;
}
static bool advance_and_accept_template_char_in_interpolation(TSLexer *lexer) {
advance(lexer);
lexer->result_symbol = TEMPLATE_CHAR_IN_INTERPOLATION;
return true;
static bool accept_and_advance(TSLexer *lexer, enum TokenType token) {
advance(lexer);
return accept_inplace(lexer, token);
}
static bool consume_wxdigit(TSLexer *lexer) {
advance(lexer);
return iswxdigit(lexer->lookahead);
advance(lexer);
return iswxdigit(lexer->lookahead);
}
// scan escape sequences \n \t \r \\ \" \uHHHH or \UHHHHHHHHA where H is a hex digit
// assumes that the leading character is '\'
static bool scan_backslash_escape_sequence(TSLexer *lexer) {
advance(lexer);
switch (lexer->lookahead) {
case '"':
case 'n':
case 'r':
case 't':
case '\\':
return advance_and_accept_escape_sequence(lexer);
case 'u':
for (int i = 0; i < 4; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return advance_and_accept_escape_sequence(lexer);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return advance_and_accept_escape_sequence(lexer);
default:
return false;
typedef struct Scanner {
bool in_template_interpolation;
bool in_quoted_context;
int template_interpolation_depth;
int quoted_context_depth;
} Scanner;
void print_debug_info(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
printf("\nDEBUG INFO START\n\n");
printf("currently at: '%c'\n\n", lexer->lookahead);
printf("could be one of\n");
printf("quoted_template_start: %x\n", valid_symbols[QUOTED_TEMPLATE_START]);
printf("quoted_template_end: %x\n", valid_symbols[QUOTED_TEMPLATE_END]);
printf("template_literal_chunk: %x", valid_symbols[TEMPLATE_LITERAL_CHUNK]);
printf("template_interpolation_start: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_START]);
printf("template_interpolation_end: %x\n", valid_symbols[TEMPLATE_INTERPOLATION_END]);
printf("\n");
printf("scanner state:\n");
printf("in_template_interpolation %x\n", scanner->in_template_interpolation);
printf("in_quoted_context %x\n", scanner->in_quoted_context);
printf("template_interpolation_depth %x\n", scanner->template_interpolation_depth);
printf("quoted_context_depth %x\n", scanner->quoted_context_depth);
printf("\nDEBUG INFO END\n\n");
}
// some helpers to keep track of template depth
void scanner_enter_interpolation_context(Scanner *scanner) {
scanner->template_interpolation_depth++;
scanner->in_template_interpolation = true;
scanner->in_quoted_context = false;
}
void scanner_exit_interpolation_context(Scanner *scanner) {
scanner->template_interpolation_depth--;
scanner->in_template_interpolation = false;
scanner->in_quoted_context = true;
}
void scanner_enter_quoted_context(Scanner *scanner) {
scanner->quoted_context_depth++;
scanner->in_quoted_context = true;
scanner->in_template_interpolation = false;
}
void scanner_exit_quoted_context(Scanner *scanner) {
scanner->quoted_context_depth--;
scanner->in_quoted_context = false;
// check if we are reentering a nested template interpolation
if (scanner->template_interpolation_depth > 0) {
scanner->in_template_interpolation = true;
}
}
// may accept multiple characters like %% as a 'template literal chunk'
// assumes that the leading character is '%' or '$'
static bool scan_template_literal_chunk_or_template_escape_sequence(TSLexer *lexer) {
const leading_char = lexer->lookahead;
advance(lexer);
// reject %{ because its the start of template directives
if (lexer->lookahead == '{') return false;
if (lexer->lookahead == leading_char) {
advance(lexer);
// accept %%{ as escape sequence
if (lexer->lookahead == '{') return advance_and_accept_escape_sequence(lexer);
}
// accept % and %% as template chars
return accept_template_char_inplace(lexer);
}
static bool scan_template_char_or_escape_sequence(TSLexer *lexer) {
switch (lexer->lookahead) {
case '"':
bool must_escape_in_nested_quoted_context(char c) {
switch (c) {
case '\n':
case '\r':
case '\t':
return false;
case '\\':
return scan_backslash_escape_sequence(lexer);
case '$':
case '%':
return scan_template_literal_chunk_or_template_escape_sequence(lexer);
return true;
default:
return advance_and_accept_template_char(lexer);
return false;
}
}
static bool scan_template_char_in_interpolation_or_escape_sequence(TSLexer *lexer) {
switch (lexer->lookahead) {
case '"':
case '\n':
case '\r':
case '\t':
// no template interpolation chars are allowed in template interpolations ( even escaped )
case '$':
case '%':
case '~':
return false;
// '}' ends the template interpolation
case '}':
return false;
case '\\':
return scan_backslash_escape_sequence(lexer);
default:
return advance_and_accept_template_char_in_interpolation(lexer);
bool scanner_scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
// literal newlines are not allowed inside a quoted context
if (must_escape_in_nested_quoted_context(lexer->lookahead) && scanner->quoted_context_depth > 0) {
return false;
}
}
while (iswspace(lexer->lookahead) && !scanner->in_quoted_context) skip(lexer);
// scanner
void *tree_sitter_hcl_external_scanner_create() { return NULL; }
void tree_sitter_hcl_external_scanner_destroy(void *p) {}
void tree_sitter_hcl_external_scanner_reset(void *p) {}
unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) { return 0; }
void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
bool tree_sitter_hcl_external_scanner_scan(
void *p,
TSLexer *lexer,
const bool *valid_symbols
) {
// when scanning string literals or quoted template literals that are not in an template directive
if (valid_symbols[TEMPLATE_CHAR] && valid_symbols[ESCAPE_SEQUENCE]) {
return scan_template_char_or_escape_sequence(lexer);
// manage quoted context
if (valid_symbols[QUOTED_TEMPLATE_START] && lexer->lookahead == '"') {
scanner_enter_quoted_context(scanner);
return accept_and_advance(lexer, QUOTED_TEMPLATE_START);
}
// quoted template literals currently inside a template directive
if (valid_symbols[TEMPLATE_CHAR_IN_INTERPOLATION] && valid_symbols[ESCAPE_SEQUENCE]) {
return scan_template_char_in_interpolation_or_escape_sequence(lexer);
if (valid_symbols[QUOTED_TEMPLATE_END] && lexer->lookahead == '"') {
scanner_exit_quoted_context(scanner);
return accept_and_advance(lexer, QUOTED_TEMPLATE_END);
}
if (valid_symbols[TEMPLATE_INTERPOLATION_START] && lexer->lookahead == '$') {
advance(lexer);
if (lexer->lookahead == '{') {
scanner_enter_interpolation_context(scanner);
return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_START);
}
if (valid_symbols[TEMPLATE_LITERAL_CHUNK]) {
// try to scan escape sequence
if (lexer->lookahead == '$') {
advance(lexer);
if (lexer->lookahead == '{') {
// $${
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
}
}
return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK);
}
}
if (valid_symbols[TEMPLATE_INTERPOLATION_END] && lexer->lookahead == '}') {
scanner_exit_interpolation_context(scanner);
return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_END);
}
// handle escape sequences in direct surrounding quoted contexts
if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && scanner->quoted_context_depth > 0) {
switch (lexer->lookahead) {
case '"':
case '\n':
case '\r':
case '\t':
return false;
case '\\':
advance(lexer);
switch (lexer->lookahead) {
case '"':
case 'n':
case 'r':
case 't':
case '\\':
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'u':
for (int i = 0; i < 4; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
default:
return false;
}
case '$':
// handled above
default:
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
}
}
return false;
}
void *tree_sitter_hcl_external_scanner_create() {
Scanner *scanner = (Scanner*)malloc(sizeof(Scanner));
if (scanner) {
scanner->in_template_interpolation = false;
scanner->in_quoted_context = false;
scanner->template_interpolation_depth = 0;
scanner->quoted_context_depth = 0;
}
return scanner;
}
void tree_sitter_hcl_external_scanner_destroy(void *p) {
free(p);
}
unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) {
memcpy(b, p, sizeof(Scanner)); return sizeof(Scanner);
}
void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b, unsigned n) {
memcpy(p, b, n);
}
bool tree_sitter_hcl_external_scanner_scan(void *p, TSLexer *lexer, const bool *valid_symbols) {
return scanner_scan((Scanner*)p, lexer, valid_symbols);
}