[Tolk] Compiler built-in __expect_type() for testing purposes

Currently, tolk-tester can test various "output" of the compiler:
pass input and check output, validate fif codegen, etc.
But it can not test compiler internals and AST representation.

I've added an ability to have special functions to check/expose
internal compiler state. The first (and the only now) is:
> __expect_type(some_expr, "<type>");
Such a call has special treatment in a compilation process.
Compilation fails if this expression doesn't have requested type.

It's intended to be used in tests only. Not present in stdlib.

tolk-tester/tests/a6.tolk

@@ -1,6 +1,9 @@
 fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) {
   // solve a 2x2 linear equation
   var D: int = a*d - b*c;;;; var Dx: int = e*d-b*f ;;;;  var Dy: int = a * f - e * c;
+  __expect_type(D, "int");
+  __expect_type(D*D, "int");
+  __expect_type(calc_phi, "() -> int");
   return (Dx/D,Dy/D);
 };;;;
 

tolk-tester/tests/generics-1.tolk

@@ -41,10 +41,12 @@ fun manyEq<T1, T2, T3>(a: T1, b: T2, c: T3): [T1, T2, T3] {
 
 @method_id(104)
 fun test104(f: int) {
-    return (
+    var result = (
         manyEq(1 ? 1 : 1, f ? 0 : null, !f ? getTwo() as int : null),
         manyEq((f ? null as int : eq2(2), beginCell().storeBool(true).endCell().beginParse().loadBool()), 0, eq4(f))
     );
+    __expect_type(result, "([int, int, int], [(int, bool), int, int])");
+    return result;
 }
 
 fun calcSum<X>(x: X, y: X) { return x + y; }
@@ -68,6 +70,7 @@ fun abstractTransform<X, Y, R>(xToY: (X) -> Y, yToR: (((Y))) -> R, initialX: X):
 @method_id(106)
 fun test106() {
     var c = beginCell().storeInt(106, 32).endCell();
+    __expect_type(calcYPlus1<int>, "(int) -> int");
     return [
         abstractTransform(cellToSlice, calcLoad32, c),
         abstractTransform(calcYPlus1<int>, calcYPlus1<int>, 0),

tolk-tester/tests/inference-tests.tolk

@@ -0,0 +1,94 @@
+// the goal of this file is not only to @testcase results —
+// but to check that this file compiles
+
+fun eq<X>(value: X): X { return value; }
+
+fun test1(x: int, y: int) {
+    __expect_type(0, "int");
+    __expect_type("0"c, "int");
+    __expect_type(x, "int");
+    __expect_type(x + y, "int");
+    __expect_type(x * y, "int");
+    __expect_type(x & y, "int");
+    __expect_type(x << y, "int");
+    __expect_type((((x))), "int");
+    __expect_type(x = x, "int");
+    __expect_type(x += x, "int");
+    __expect_type(x &= x, "int");
+    __expect_type(random() ? x : y, "int");
+    __expect_type(eq(x), "int");
+    __expect_type(eq<int>(x), "int");
+    __expect_type(eq<int>(null), "int");
+    __expect_type(x as int, "int");
+    __expect_type(+x, "int");
+    __expect_type(~x, "int");
+    {
+        var x: slice = beginCell().endCell().beginParse();
+        __expect_type(x, "slice");
+        __expect_type(beginCell(), "builder");
+        __expect_type(beginCell().endCell(), "cell");
+    }
+}
+
+fun test2(x: int, y: bool) {
+    __expect_type(!x, "bool");
+    __expect_type(x != x, "bool");
+    __expect_type(x <= x, "bool");
+    __expect_type(x <=> x, "bool");
+    __expect_type(x <=> x, "bool");
+    __expect_type(!random(), "bool");
+    __expect_type(!!(x != null), "bool");
+    __expect_type(x ? x != null : null == x, "bool");
+    __expect_type(y & true, "bool");
+    __expect_type(y ^= false, "bool");
+    __expect_type(x && y, "bool");
+    __expect_type(true && false && true, "bool");
+    __expect_type(x || x, "bool");
+    __expect_type(x || !x || (true & false), "bool");
+}
+
+fun test3() {
+    __expect_type(true as int, "int");
+    __expect_type(!random() as int, "int");
+}
+
+fun test4(x: int) {
+    __expect_type((), "()");
+    __expect_type((x, x), "(int, int)");
+    __expect_type((x, (x, x), x), "(int, (int, int), int)");
+}
+
+fun test5(x: int) {
+    __expect_type([], "[]");
+    __expect_type([x], "[int]");
+    __expect_type([x, x >= 1], "[int, bool]");
+    __expect_type([x, x >= 1, null as slice], "[int, bool, slice]");
+    __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])");
+    __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, cell]");
+}
+
+fun test6() {
+    var t = createEmptyTuple();
+    __expect_type(t, "tuple");
+    t.tuplePush(1);
+    __expect_type(t, "tuple");
+}
+
+fun test7() {
+    __expect_type(test3(), "void");
+    __expect_type(test3, "() -> void");
+    var cb = test1;
+    __expect_type(cb, "(int, int) -> void");
+    var t = createEmptyTuple();
+    __expect_type(beginCell().endCell, "(builder) -> cell");
+    // __expect_type(eq<(int, slice)>, "(int, slice) -> (int, slice)");
+}
+
+
+fun main() {
+    return 0;
+}
+
+/**
+@testcase | 0 | | 0
+*/

tolk/builtins.cpp

@@ -1255,6 +1255,13 @@ void define_builtins() {
   define_builtin_func("debugDumpStack", {}, Unit, nullptr,
                                 AsmOp::Custom("DUMPSTK", 0, 0),
                                 0);
+
+  // functions not presented in stdlib at all
+  // used in tolk-tester to check/expose internal compiler state
+  // each of them is handled in a special way, search by its name
+  define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr,
+                                AsmOp::Nop(),
+                                FunctionData::flagMarkedAsPure);
 }
 
 }  // namespace tolk

tolk/lexer.cpp

@@ -555,6 +555,15 @@ Lexer::Lexer(const SrcFile* file)
   next();
 }
 
+Lexer::Lexer(std::string_view text)
+  : file(nullptr)
+  , p_start(text.data())
+  , p_end(p_start + text.size())
+  , p_next(p_start)
+  , location() {
+  next();
+}
+
 void Lexer::next() {
   while (cur_token_idx == last_token_idx && !is_eof()) {
     update_location();
@@ -563,7 +572,7 @@ void Lexer::next() {
     }
   }
   if (is_eof()) {
-    add_token(tok_eof, file->text);
+    add_token(tok_eof, "");
   }
   cur_token = tokens_circularbuf[++cur_token_idx & 7];
 }

tolk/lexer.h

@@ -173,6 +173,7 @@ public:
   };
 
   explicit Lexer(const SrcFile* file);
+  explicit Lexer(std::string_view text);
   Lexer(const Lexer&) = delete;
   Lexer &operator=(const Lexer&) = delete;
 

tolk/pipe-infer-types-and-calls.cpp

@@ -240,6 +240,24 @@ public:
   TypePtr get_result() const { return unified_result; }
 };
 
+// handle __expect_type(expr, "type") call
+// this is used in compiler tests
+GNU_ATTRIBUTE_NOINLINE GNU_ATTRIBUTE_COLD
+static void handle_possible_compiler_internal_call(const FunctionData* current_function, V<ast_function_call> v) {
+  const FunctionData* fun_ref = v->fun_maybe;
+  tolk_assert(fun_ref && fun_ref->is_builtin_function());
+  static_cast<void>(current_function);
+
+  if (fun_ref->name == "__expect_type") {
+    tolk_assert(v->get_num_args() == 2);
+    TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as<ast_string_const>()->str_val);
+    TypePtr expr_type = v->get_arg(0)->inferred_type;
+    if (expected_type != expr_type) {
+      v->error("__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type));
+    }
+  }
+}
+
 /*
  * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type.
  * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types.
@@ -974,6 +992,9 @@ class InferCheckTypesAndCallsAndFieldsVisitor final {
     TypePtr inferred_type = dot_obj && fun_ref->does_return_self() ? dot_obj->inferred_type : fun_ref->inferred_return_type;
     assign_inferred_type(v, inferred_type);
     assign_inferred_type(callee, fun_ref->inferred_full_type);
+    if (fun_ref->is_builtin_function() && fun_ref->name[0] == '_') {
+      handle_possible_compiler_internal_call(current_function, v);
+    }
     // note, that mutate params don't affect typing, they are handled when converting to IR
   }
 

tolk/type-system.cpp

@@ -651,11 +651,6 @@ static TypePtr parse_simple_type(Lexer& lex) {
       std::vector<TypePtr> items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`");
       return TypeDataTypedTuple::create(std::move(items));
     }
-    case tok_fun: {
-      lex.next();
-      std::vector<TypePtr> params_types = parse_nested_type_list_in_parenthesis(lex);
-      lex.expect(tok_arrow, "`->`");
-    }
     default:
       lex.unexpected("<type>");
   }
@@ -695,6 +690,12 @@ TypePtr parse_type_from_tokens(Lexer& lex) {
   return parse_type_expression(lex);
 }
 
+// for internal usage only
+TypePtr parse_type_from_string(std::string_view text) {
+  Lexer lex(text);
+  return parse_type_expression(lex);
+}
+
 std::ostream& operator<<(std::ostream& os, TypePtr type_data) {
   return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)");
 }

tolk/type-system.h

@@ -417,6 +417,7 @@ public:
 
 class Lexer;
 TypePtr parse_type_from_tokens(Lexer& lex);
+TypePtr parse_type_from_string(std::string_view text);
 
 void type_system_init();