jenterkin/tree-sitter-jhcl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

misc: rework shell.nix, reformat, ditch .editorconfig

Browse Source
This commit is contained in:
Michael Hoffmann
2023-04-07 12:17:28 +02:00
parent 0ff887f2a6
commit b8fcca985c
5 changed files with 92 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.editorconfig
View File

@@ -1,10 +0,0 @@
root = true
[*.{cc,txt,js}]
indent_style = space
indent_size = 2
tab_width = 8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8

3
CHANGELOG.md
View File

@@ -5,6 +5,9 @@
feature
* add dialects so we can have different queries in `nvim-treesitter`
housekeeping:
* reformat using LSPs, ditch editorconfig
## 1.0.0 - 2022-12-02
breaking:

2
make_grammar.js
View File

@@ -287,7 +287,7 @@ module.exports = function make_grammar(dialect) {
$.template_interpolation,
$.template_directive,
$.template_literal,
)),
)),
template_literal: $ => prec.right(repeat1(
$._template_literal_chunk,

25
shell.nix
View File

@@ -1,19 +1,12 @@
let
nixpkgs = fetchTarball {
name = "nixpkgs";
url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/21.05.tar.gz";
sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
};
pkgs = import nixpkgs {};
pkgs = import <nixpkgs> { };
in
pkgs.mkShell {
name = "env";
buildInputs = with pkgs; [
nodejs
];
shellHook = ''
PATH=./node_modules/.bin:$PATH
command -v tree-sitter >/dev/null 2>&1 || npm install tree-sitter-cli@0.20.6
'';
}
pkgs.mkShell {
name = "env";
buildInputs = with pkgs; [
nodejs
tree-sitter
emscripten
];
}

162
src/scanner.cc
View File

@@ -1,15 +1,15 @@
#include <tree_sitter/parser.h>
#include <climits>
#include <vector>
#include <string>
#include <wctype.h>
#include <assert.h>
#include <climits>
#include <string>
#include <vector>
#include <wctype.h>
namespace {
using std::vector;
using std::string;
using std::vector;
enum TokenType {
QUOTED_TEMPLATE_START,
@@ -39,8 +39,7 @@ struct Context {
struct Scanner {
public:
unsigned serialize(char* buf) {
unsigned serialize(char *buf) {
unsigned size = 0;
if (context_stack.size() > CHAR_MAX) {
@@ -48,8 +47,10 @@ public:
}
buf[size++] = context_stack.size();
for (vector<Context>::iterator it = context_stack.begin(); it != context_stack.end(); ++it) {
if (size + 2 + it->heredoc_identifier.size() >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
for (vector<Context>::iterator it = context_stack.begin();
it != context_stack.end(); ++it) {
if (size + 2 + it->heredoc_identifier.size() >=
TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
return 0;
}
if (it->heredoc_identifier.size() > CHAR_MAX) {
@@ -63,7 +64,7 @@ public:
return size;
}
void deserialize(const char* buf, unsigned n) {
void deserialize(const char *buf, unsigned n) {
context_stack.clear();
if (n == 0) {
@@ -76,14 +77,15 @@ public:
Context ctx;
ctx.type = static_cast<ContextType>(buf[size++]);
uint8_t heredoc_identifier_size = buf[size++];
ctx.heredoc_identifier.assign(buf + size, buf + size + heredoc_identifier_size);
ctx.heredoc_identifier.assign(buf + size,
buf + size + heredoc_identifier_size);
size += heredoc_identifier_size;
context_stack.push_back(ctx);
}
assert(size == n);
}
bool scan(TSLexer* lexer, const bool* valid_symbols) {
bool scan(TSLexer *lexer, const bool *valid_symbols) {
bool has_leading_whitespace_with_newline = false;
while (iswspace(lexer->lookahead)) {
if (lexer->lookahead == '\n') {
@@ -95,26 +97,25 @@ public:
return false;
}
// manage quoted context
if (valid_symbols[QUOTED_TEMPLATE_START] && !in_quoted_context() && lexer->lookahead == '"') {
Context ctx = { QUOTED_TEMPLATE, "" };
if (valid_symbols[QUOTED_TEMPLATE_START] && !in_quoted_context() &&
lexer->lookahead == '"') {
Context ctx = {QUOTED_TEMPLATE, ""};
context_stack.push_back(ctx);
return accept_and_advance(lexer, QUOTED_TEMPLATE_START);
}
if (valid_symbols[QUOTED_TEMPLATE_END] && in_quoted_context() && lexer->lookahead == '"') {
if (valid_symbols[QUOTED_TEMPLATE_END] && in_quoted_context() &&
lexer->lookahead == '"') {
context_stack.pop_back();
return accept_and_advance(lexer, QUOTED_TEMPLATE_END);
}
// manage template interpolations
if (
valid_symbols[TEMPLATE_INTERPOLATION_START] &&
valid_symbols[TEMPLATE_LITERAL_CHUNK] &&
!in_interpolation_context() &&
lexer->lookahead == '$'
) {
if (valid_symbols[TEMPLATE_INTERPOLATION_START] &&
valid_symbols[TEMPLATE_LITERAL_CHUNK] && !in_interpolation_context() &&
lexer->lookahead == '$') {
advance(lexer);
if (lexer->lookahead == '{') {
Context ctx = { TEMPLATE_INTERPOLATION, "" };
Context ctx = {TEMPLATE_INTERPOLATION, ""};
context_stack.push_back(ctx);
return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_START);
}
@@ -128,21 +129,19 @@ public:
}
return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK);
}
if (valid_symbols[TEMPLATE_INTERPOLATION_END] && in_interpolation_context() && lexer->lookahead == '}') {
if (valid_symbols[TEMPLATE_INTERPOLATION_END] &&
in_interpolation_context() && lexer->lookahead == '}') {
context_stack.pop_back();
return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_END);
}
// manage template directives
if (
valid_symbols[TEMPLATE_DIRECTIVE_START] &&
valid_symbols[TEMPLATE_LITERAL_CHUNK] &&
!in_directive_context() &&
lexer->lookahead == '%'
) {
if (valid_symbols[TEMPLATE_DIRECTIVE_START] &&
valid_symbols[TEMPLATE_LITERAL_CHUNK] && !in_directive_context() &&
lexer->lookahead == '%') {
advance(lexer);
if (lexer->lookahead == '{') {
Context ctx = { TEMPLATE_DIRECTIVE, "" };
Context ctx = {TEMPLATE_DIRECTIVE, ""};
context_stack.push_back(ctx);
return accept_and_advance(lexer, TEMPLATE_DIRECTIVE_START);
}
@@ -156,7 +155,8 @@ public:
}
return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK);
}
if (valid_symbols[TEMPLATE_DIRECTIVE_END] && in_directive_context() && lexer->lookahead == '}') {
if (valid_symbols[TEMPLATE_DIRECTIVE_END] && in_directive_context() &&
lexer->lookahead == '}') {
context_stack.pop_back();
return accept_and_advance(lexer, TEMPLATE_DIRECTIVE_END);
}
@@ -165,18 +165,21 @@ public:
if (valid_symbols[HEREDOC_IDENTIFIER] && !in_heredoc_context()) {
string identifier;
// TODO: check that this is a valid identifier
while (iswalnum(lexer->lookahead) || lexer->lookahead == '_' || lexer->lookahead == '-') {
while (iswalnum(lexer->lookahead) || lexer->lookahead == '_' ||
lexer->lookahead == '-') {
identifier.push_back(lexer->lookahead);
advance(lexer);
}
Context ctx = { HEREDOC_TEMPLATE, identifier };
Context ctx = {HEREDOC_TEMPLATE, identifier};
context_stack.push_back(ctx);
return accept_inplace(lexer, HEREDOC_IDENTIFIER);
}
if (valid_symbols[HEREDOC_IDENTIFIER] && in_heredoc_context() && has_leading_whitespace_with_newline) {
if (valid_symbols[HEREDOC_IDENTIFIER] && in_heredoc_context() &&
has_leading_whitespace_with_newline) {
string expected_identifier = context_stack.back().heredoc_identifier;
for (string::iterator it = expected_identifier.begin(); it != expected_identifier.end(); ++it) {
for (string::iterator it = expected_identifier.begin();
it != expected_identifier.end(); ++it) {
if (lexer->lookahead == *it) {
advance(lexer);
} else {
@@ -186,7 +189,7 @@ public:
// check if the identifier is on a line of its own
lexer->mark_end(lexer);
while (iswspace(lexer->lookahead) && lexer->lookahead != '\n') {
advance(lexer);
advance(lexer);
}
if (lexer->lookahead == '\n') {
context_stack.pop_back();
@@ -202,31 +205,32 @@ public:
// handle template literal chunks in quoted contexts
//
// they may not contain newlines and may contain escape sequences
if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && in_quoted_context()) {
switch (lexer->lookahead) {
case '\\':
advance(lexer);
switch (lexer->lookahead) {
case '"':
case 'n':
case 'r':
case 't':
case '\\':
advance(lexer);
switch (lexer->lookahead) {
case '"':
case 'n':
case 'r':
case 't':
case '\\':
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'u':
for (int i = 0; i < 4; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
default:
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'u':
for (int i = 0; i < 4; i++) {
if (!consume_wxdigit(lexer))
return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer))
return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
default:
return false;
}
}
}
@@ -242,23 +246,21 @@ public:
private:
vector<Context> context_stack;
void advance(TSLexer* lexer) {
lexer->advance(lexer, false);
}
void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
void skip(TSLexer* lexer) { lexer->advance(lexer, true); }
void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
bool accept_inplace(TSLexer* lexer, TokenType token) {
bool accept_inplace(TSLexer *lexer, TokenType token) {
lexer->result_symbol = token;
return true;
}
bool accept_and_advance(TSLexer* lexer, TokenType token) {
bool accept_and_advance(TSLexer *lexer, TokenType token) {
advance(lexer);
return accept_inplace(lexer, token);
}
bool consume_wxdigit(TSLexer* lexer) {
bool consume_wxdigit(TSLexer *lexer) {
advance(lexer);
return iswxdigit(lexer->lookahead);
}
@@ -270,13 +272,9 @@ private:
return context_stack.back().type == type;
}
bool in_quoted_context() {
return in_context_type(QUOTED_TEMPLATE);
}
bool in_quoted_context() { return in_context_type(QUOTED_TEMPLATE); }
bool in_heredoc_context() {
return in_context_type(HEREDOC_TEMPLATE);
}
bool in_heredoc_context() { return in_context_type(HEREDOC_TEMPLATE); }
bool in_template_context() {
return in_quoted_context() || in_heredoc_context();
@@ -286,9 +284,7 @@ private:
return in_context_type(TEMPLATE_INTERPOLATION);
}
bool in_directive_context() {
return in_context_type(TEMPLATE_DIRECTIVE);
}
bool in_directive_context() { return in_context_type(TEMPLATE_DIRECTIVE); }
};
} // namespace
@@ -296,27 +292,27 @@ private:
extern "C" {
// tree sitter callbacks
void* tree_sitter_hcl_external_scanner_create() {
return new Scanner();
}
void *tree_sitter_hcl_external_scanner_create() { return new Scanner(); }
void tree_sitter_hcl_external_scanner_destroy(void* p) {
Scanner* scanner = static_cast<Scanner*>(p);
void tree_sitter_hcl_external_scanner_destroy(void *p) {
Scanner *scanner = static_cast<Scanner *>(p);
delete scanner;
}
unsigned tree_sitter_hcl_external_scanner_serialize(void* p, char* b) {
Scanner* scanner = static_cast<Scanner*>(p);
unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) {
Scanner *scanner = static_cast<Scanner *>(p);
return scanner->serialize(b);
}
void tree_sitter_hcl_external_scanner_deserialize(void* p, const char* b, unsigned n) {
Scanner* scanner = static_cast<Scanner*>(p);
void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b,
unsigned n) {
Scanner *scanner = static_cast<Scanner *>(p);
return scanner->deserialize(b, n);
}
bool tree_sitter_hcl_external_scanner_scan(void* p, TSLexer* lexer, const bool* valid_symbols) {
Scanner* scanner = static_cast<Scanner*>(p);
bool tree_sitter_hcl_external_scanner_scan(void *p, TSLexer *lexer,
const bool *valid_symbols) {
Scanner *scanner = static_cast<Scanner *>(p);
return scanner->scan(lexer, valid_symbols);
}