external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

[Tolk] Rewrite lexer, spaces are not mandatory anymore

Browse source 
A new lexer is noticeably faster and memory efficient
(although splitting a file to tokens is negligible in a whole pipeline).

But the purpose of rewriting lexer was not just to speed up,
but to allow writing code without spaces:
`2+2` is now 4, not a valid identifier as earlier.

The variety of symbols allowed in identifier has greatly reduced
and is now similar to other languages.

SrcLocation became 8 bytes on stack everywhere.

Command-line flags were also reworked:
- the input for Tolk compiler is only a single file now, it's parsed, and parsing continues while new #include are resolved
- flags like -A -P and so on are no more needed, actually
This commit is contained in:
tolk-vm 2024-10-31 10:59:23 +04:00
parent 0bcc0b3c12
commit f0e6470d0b
No known key found for this signature in database
GPG key ID: 7905DD7FE0324B12
26 changed files with 2042 additions and 2129 deletions
Download patch file Download diff file Expand all files Collapse all files

77
tolk/tolk-wasm.cpp
View file

@ -31,81 +31,58 @@
#include "td/utils/Status.h"
#include <sstream>
#include <iomanip>
#include "vm/boc.h"
td::Result<std::string> compile_internal(char *config_json) {
TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))
auto &obj = input_json.get_object();
td::JsonObject& config = input_json.get_object();
TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false));
TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false));
auto &sources_arr = sources_obj.get_array();
std::vector<std::string> sources;
for (auto &item : sources_arr) {
sources.push_back(item.get_string().str());
}
TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optimizationLevel", true, 2));
TRY_RESULT(stack_comments, td::get_json_object_bool_field(config, "withStackComments", true, false));
TRY_RESULT(entrypoint_file_name, td::get_json_object_string_field(config, "entrypointFileName", false));
tolk::opt_level = std::max(0, opt_level);
tolk::program_envelope = true;
tolk::verbosity = 0;
tolk::indent = 1;
tolk::stack_layout_comments = stack_comments;
std::ostringstream outs, errs;
auto compile_res = tolk::tolk_proceed(sources, outs, errs);
if (compile_res != 0) {
return td::Status::Error(std::string("Tolk compilation error: ") + errs.str());
int tolk_res = tolk::tolk_proceed(entrypoint_file_name, outs, errs);
if (tolk_res != 0) {
return td::Status::Error("Tolk compilation error: " + errs.str());
}
TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false));
TRY_RESULT(boc, vm::std_boc_serialize(code_cell));
TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/"));
td::JsonBuilder result_json;
auto result_obj = result_json.enter_object();
result_obj("status", "ok");
result_obj("codeBoc", td::base64_encode(boc));
result_obj("fiftCode", outs.str());
result_obj("codeHashHex", code_cell->get_hash().to_hex());
result_obj.leave();
outs.clear();
errs.clear();
auto obj = result_json.enter_object();
obj("status", "ok");
obj("fiftCode", fift_res.fiftCode);
obj("codeBoc64", fift_res.codeBoc64);
obj("codeHashHex", fift_res.codeHashHex);
obj.leave();
return result_json.string_builder().as_cslice().str();
}
/// Callback used to retrieve additional source files or data.
///
/// @param _kind The kind of callback (a string).
/// @param _data The data for the callback (a string).
/// @param o_contents A pointer to the contents of the file, if found. Allocated via malloc().
/// @param o_error A pointer to an error message, if there is one. Allocated via malloc().
///
/// The callback implementor must use malloc() to allocate storage for
/// contents or error. The callback implementor must use free() to free
/// said storage after tolk_compile returns.
///
/// If the callback is not supported, *o_contents and *o_error must be set to NULL.
typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error);
/// Callback used to retrieve file contents from a "not file system". See tolk-js for implementation.
/// The callback must fill either destContents or destError.
/// The implementor must use malloc() for them and use free() after tolk_compile returns.
typedef void (*CStyleReadFileCallback)(int kind, char const* data, char** destContents, char** destError);
tolk::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback)
{
tolk::ReadCallback::Callback readCallback;
if (_readCallback) {
readCallback = [=](tolk::ReadCallback::Kind _kind, char const* _data) -> td::Result<std::string> {
char* contents_c = nullptr;
char* error_c = nullptr;
_readCallback(tolk::ReadCallback::kindString(_kind).data(), _data, &contents_c, &error_c);
if (!contents_c && !error_c) {
readCallback = [=](tolk::ReadCallback::Kind kind, char const* data) -> td::Result<std::string> {
char* destContents = nullptr;
char* destError = nullptr;
_readCallback(static_cast<int>(kind), data, &destContents, &destError);
if (!destContents && !destError) {
return td::Status::Error("Callback not supported");
}
if (contents_c) {
return contents_c;
if (destContents) {
return destContents;
}
return td::Status::Error(std::string(error_c));
return td::Status::Error(std::string(destError));
};
}
return readCallback;