[Tolk] Rewrite lexer, spaces are not mandatory anymore

A new lexer is noticeably faster and memory efficient
(although splitting a file to tokens is negligible in a whole pipeline).

But the purpose of rewriting lexer was not just to speed up,
but to allow writing code without spaces:
`2+2` is now 4, not a valid identifier as earlier.

The variety of symbols allowed in identifier has greatly reduced
and is now similar to other languages.

SrcLocation became 8 bytes on stack everywhere.

Command-line flags were also reworked:
- the input for Tolk compiler is only a single file now, it's parsed, and parsing continues while new #include are resolved
- flags like -A -P and so on are no more needed, actually

crypto/fift/utils.cpp

@@ -211,20 +211,39 @@ td::Result<fift::SourceLookup> create_mem_source_lookup(std::string main, std::s
                               fift_dir);
 }
 
-td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) {
+td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code) {
   std::stringstream ss;
   std::string sb;
   sb.reserve(asm_code.size() + 100);
-  sb.append("\"Asm.fif\" include\n ");
-  sb.append(is_raw ? "<{" : "");
+  sb.append("\"Asm.fif\" include\n <{\n");
   sb.append(asm_code.data(), asm_code.size());
-  sb.append(is_raw ? "}>c" : "");
-  sb.append(" boc>B \"res\" B>file");
+  sb.append("\n}>c boc>B \"res\" B>file");
 
-  TRY_RESULT(source_lookup, create_source_lookup(std::move(sb), true, true, true, false, false, false, false, fift_dir));
+  TRY_RESULT(source_lookup, create_source_lookup(std::move(sb), true, true, true, false, false, false, false));
   TRY_RESULT(res, run_fift(std::move(source_lookup), &ss));
   TRY_RESULT(boc, res.read_file("res"));
   return vm::std_boc_deserialize(std::move(boc.data));
 }
 
+td::Result<CompiledProgramOutput> compile_asm_program(std::string&& program_code, const std::string& fift_dir) {
+  std::string main_fif;
+  main_fif.reserve(program_code.size() + 100);
+  main_fif.append(program_code.data(), program_code.size());
+  main_fif.append(R"( dup hashB B>X      $>B "hex" B>file)");   // write codeHashHex to a file
+  main_fif.append(R"(     boc>B B>base64 $>B "boc" B>file)");   // write codeBoc64 to a file
+
+  std::stringstream fift_output_stream;
+  TRY_RESULT(source_lookup, create_source_lookup(std::move(main_fif), true, true, false, false, false, false, false, fift_dir));
+  TRY_RESULT(res, run_fift(std::move(source_lookup), &fift_output_stream));
+
+  TRY_RESULT(boc, res.read_file("boc"));
+  TRY_RESULT(hex, res.read_file("hex"));
+
+  return CompiledProgramOutput{
+    std::move(program_code),
+    std::move(boc.data),
+    std::move(hex.data),
+  };
+}
+
 }  // namespace fift

crypto/fift/utils.h

@@ -26,11 +26,21 @@ struct FiftOutput {
   SourceLookup source_lookup;
   std::string output;
 };
+
+// given a valid Fift code PROGRAM{ ... }END>c, compile_asm_program() returns this output
+// now it's used primarily for wasm output (see tolk-js, for example)
+struct CompiledProgramOutput {
+  std::string fiftCode;
+  std::string codeBoc64;
+  std::string codeHashHex;
+};
+
 td::Result<fift::SourceLookup> create_mem_source_lookup(std::string main, std::string fift_dir = "",
                                                         bool need_preamble = true, bool need_asm = true,
                                                         bool need_ton_util = true, bool need_lisp = true,
                                                         bool need_w3_code = true);
 td::Result<FiftOutput> mem_run_fift(std::string source, std::vector<std::string> args = {}, std::string fift_dir = "");
 td::Result<FiftOutput> mem_run_fift(SourceLookup source_lookup, std::vector<std::string> args);
-td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true);
+td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code);
+td::Result<CompiledProgramOutput> compile_asm_program(std::string&& program_code, const std::string& fift_dir);
 }  // namespace fift

crypto/funcfiftlib/funcfiftlib.cpp

@@ -37,10 +37,10 @@
 
 td::Result<std::string> compile_internal(char *config_json) {
   TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))
-  auto &obj = input_json.get_object();
+  td::JsonObject& config = input_json.get_object();
 
-  TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false));
-  TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false));
+  TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optLevel", false));
+  TRY_RESULT(sources_obj, td::get_json_object_field(config, "sources", td::JsonValue::Type::Array, false));
 
   auto &sources_arr = sources_obj.get_array();
 
@@ -52,29 +52,25 @@ td::Result<std::string> compile_internal(char *config_json) {
 
   funC::opt_level = std::max(0, opt_level);
   funC::program_envelope = true;
+  funC::asm_preamble = true;
   funC::verbosity = 0;
   funC::indent = 1;
 
   std::ostringstream outs, errs;
-  auto compile_res = funC::func_proceed(sources, outs, errs);
-
-  if (compile_res != 0) {
-    return td::Status::Error(std::string("Func compilation error: ") + errs.str());
+  int funC_res = funC::func_proceed(sources, outs, errs);
+  if (funC_res != 0) {
+    return td::Status::Error("FunC compilation error: " + errs.str());
   }
 
-  TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false));
-  TRY_RESULT(boc, vm::std_boc_serialize(code_cell));
+  TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/"));
 
   td::JsonBuilder result_json;
-  auto result_obj = result_json.enter_object();
-  result_obj("status", "ok");
-  result_obj("codeBoc", td::base64_encode(boc));
-  result_obj("fiftCode", outs.str());
-  result_obj("codeHashHex", code_cell->get_hash().to_hex());
-  result_obj.leave();
-
-  outs.clear();
-  errs.clear();
+  auto obj = result_json.enter_object();
+  obj("status", "ok");
+  obj("fiftCode", std::move(fift_res.fiftCode));
+  obj("codeBoc", std::move(fift_res.codeBoc64));
+  obj("codeHashHex", std::move(fift_res.codeHashHex));
+  obj.leave();
 
   return result_json.string_builder().as_cslice().str();
 }

crypto/smartcont/mathlib.tolk

@@ -572,9 +572,9 @@ int atanh_f261(int x, int n) inline_ref {
     s -= 1;
   }
   x += t;
-  int 2x = 2 * x;
-  int y = lshift256divr(2x, (x >> 1) - t);
-  ;; y = 2x - (mulrshiftr256(2x, y) ~>> 2);  ;; this line could improve precision on very rare occasions
+  int `2x` = 2 * x;
+  int y = lshift256divr(`2x`, (x >> 1) - t);
+  ;; y = `2x` - (mulrshiftr256(2x, y) ~>> 2);  ;; this line could improve precision on very rare occasions
   return (atanh_f258(y, 36), s);
 }