external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

[Tolk] Rewrite lexer, spaces are not mandatory anymore

Browse source 
A new lexer is noticeably faster and memory efficient
(although splitting a file to tokens is negligible in a whole pipeline).

But the purpose of rewriting lexer was not just to speed up,
but to allow writing code without spaces:
`2+2` is now 4, not a valid identifier as earlier.

The variety of symbols allowed in identifier has greatly reduced
and is now similar to other languages.

SrcLocation became 8 bytes on stack everywhere.

Command-line flags were also reworked:
- the input for Tolk compiler is only a single file now, it's parsed, and parsing continues while new #include are resolved
- flags like -A -P and so on are no more needed, actually
This commit is contained in:
tolk-vm 2024-10-31 10:59:23 +04:00
parent 0bcc0b3c12
commit f0e6470d0b
No known key found for this signature in database
GPG key ID: 7905DD7FE0324B12
26 changed files with 2042 additions and 2129 deletions
Download patch file Download diff file Expand all files Collapse all files

28
tolk/abscode.cpp
View file

@ -24,29 +24,19 @@ namespace tolk {
*
*/
TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const SrcLocation* loc)
: v_type(_type), idx(_idx), cls(_cls), coord(0) {
TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, SrcLocation loc)
: v_type(_type), idx(_idx), cls(_cls), coord(0), where(loc) {
if (sym) {
name = sym->sym_idx;
sym->value->idx = _idx;
}
if (loc) {
where = std::make_unique<SrcLocation>(*loc);
}
if (!_type) {
v_type = TypeExpr::new_hole();
}
if (cls == _Named) {
undefined = true;
}
}
void TmpVar::set_location(const SrcLocation& loc) {
if (where) {
*where = loc;
} else {
where = std::make_unique<SrcLocation>(loc);
}
void TmpVar::set_location(SrcLocation loc) {
where = loc;
}
void TmpVar::dump(std::ostream& os) const {
@ -469,10 +459,10 @@ void CodeBlob::print(std::ostream& os, int flags) const {
if ((flags & 8) != 0) {
for (const auto& var : vars) {
var.dump(os);
if (var.where && (flags & 1) != 0) {
var.where->show(os);
if (var.where.is_defined() && (flags & 1) != 0) {
var.where.show(os);
os << " defined here:\n";
var.where->show_context(os);
var.where.show_context(os);
}
}
}
@ -483,7 +473,7 @@ void CodeBlob::print(std::ostream& os, int flags) const {
os << "-------- END ---------\n\n";
}
var_idx_t CodeBlob::create_var(int cls, TypeExpr* var_type, SymDef* sym, const SrcLocation* location) {
var_idx_t CodeBlob::create_var(int cls, TypeExpr* var_type, SymDef* sym, SrcLocation location) {
vars.emplace_back(var_cnt, cls, var_type, sym, location);
if (sym) {
sym->value->idx = var_cnt;
@ -501,7 +491,7 @@ bool CodeBlob::import_params(FormalArgList arg_list) {
SymDef* arg_sym;
SrcLocation arg_loc;
std::tie(arg_type, arg_sym, arg_loc) = par;
list.push_back(create_var(arg_sym ? (TmpVar::_In | TmpVar::_Named) : TmpVar::_In, arg_type, arg_sym, &arg_loc));
list.push_back(create_var(arg_sym ? (TmpVar::_In | TmpVar::_Named) : TmpVar::_In, arg_type, arg_sym, arg_loc));
}
emplace_back(loc, Op::_Import, list);
in_var_cnt = var_cnt;