[FunC] Enrich testing framework, add fif output patterns

* @fif_codegen to match compiled.fif against an expected pattern
* @fif_codegen_avoid to ensure compiled.fif doesn't contain a substring
* both in Python and JS run_tests
* consider tests/codegen_check_demo.fc for examples

crypto/func/auto-tests/run_tests.js

@@ -89,6 +89,9 @@ class CompareOutputError extends Error {
     }
 }
 
+class CompareFifCodegenError extends Error {
+}
+
 
 /*
  * In positive tests, there are several testcases "input X should produce output Y".
@@ -124,34 +127,136 @@ class FuncTestCaseInputOutput {
 
 /*
  * @stderr checks, when compilation fails, that stderr (compilation error) is expected.
+ * If it's multiline, all lines must be present in specified order.
  */
-class FuncTestCaseStderrIncludes {
+class FuncTestCaseStderr {
-    constructor(/**string*/ expected_substr) {
+    constructor(/**string[]*/ stderr_pattern, /**boolean*/ avoid) {
-        this.expected_substr = expected_substr
+        this.stderr_pattern = stderr_pattern
+        this.avoid = avoid
     }
 
     check(/**string*/ stderr) {
-        if (!stderr.includes(this.expected_substr))
+        const line_match = this.find_pattern_in_stderr(stderr.split(/\n/))
-            throw new CompareOutputError(`pattern '${this.expected_substr}' not found in stderr`, stderr)
+        if (line_match === -1 && !this.avoid)
+            throw new CompareOutputError("pattern not found in stderr:\n" +
+                this.stderr_pattern.map(x => "    " + x).join("\n"), stderr)
+        else if (line_match !== -1 && this.avoid)
+            throw new CompareOutputError(`pattern found (line ${line_match + 1}), but not expected to be:\n` +
+                this.stderr_pattern.map(x => "    " + x).join("\n"), stderr)
+    }
+
+    find_pattern_in_stderr(/**string[]*/ stderr) {
+        for (let line_start = 0; line_start < stderr.length; ++line_start)
+            if (this.try_match_pattern(0, stderr, line_start))
+                return line_start
+        return -1
+    }
+
+    try_match_pattern(/**number*/ pattern_offset, /**string[]*/ stderr, /**number*/ offset) {
+        if (pattern_offset >= this.stderr_pattern.length)
+            return true
+        if (offset >= stderr.length)
+            return false
+
+        const line_pattern = this.stderr_pattern[pattern_offset]
+        const line_output = stderr[offset]
+        return line_output.includes(line_pattern) && this.try_match_pattern(pattern_offset + 1, stderr, offset + 1)
     }
 }
 
+/*
+ * @fif_codegen checks that contents of compiled.fif matches the expected pattern.
+ * @fif_codegen_avoid checks that is does not match the pattern.
+ * See comments in run_tests.py.
+ */
+class FuncTestCaseFifCodegen {
+    constructor(/**string[]*/ fif_pattern, /**boolean*/ avoid) {
+        /** @type {string[]} */
+        this.fif_pattern = fif_pattern.map(s => s.trim())
+        this.avoid = avoid
+    }
+
+    check(/**string[]*/ fif_output) {
+        // in case there are no comments at all (typically for wasm), drop them from fif_pattern
+        const has_comments = fif_output.some(line => line.includes("//") && !line.includes("generated from"))
+        if (!has_comments) {
+            this.fif_pattern = this.fif_pattern.map(s => FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(s)[0])
+            this.fif_pattern = this.fif_pattern.filter(s => s !== '')
+        }
+
+        const line_match = this.find_pattern_in_fif_output(fif_output)
+        if (line_match === -1 && !this.avoid)
+            throw new CompareFifCodegenError("pattern not found:\n" +
+                this.fif_pattern.map(x => "    " + x).join("\n"))
+        else if (line_match !== -1 && this.avoid)
+            throw new CompareFifCodegenError(`pattern found (line ${line_match + 1}), but not expected to be:\n` +
+                this.fif_pattern.map(x => "    " + x).join("\n"))
+    }
+
+    find_pattern_in_fif_output(/**string[]*/ fif_output) {
+        for (let line_start = 0; line_start < fif_output.length; ++line_start)
+            if (this.try_match_pattern(0, fif_output, line_start))
+                return line_start
+        return -1
+    }
+
+    try_match_pattern(/**number*/ pattern_offset, /**string[]*/ fif_output, /**number*/ offset) {
+        if (pattern_offset >= this.fif_pattern.length)
+            return true
+        if (offset >= fif_output.length)
+            return false
+        const line_pattern = this.fif_pattern[pattern_offset]
+        const line_output = fif_output[offset]
+
+        if (line_pattern !== "...") {
+            if (!FuncTestCaseFifCodegen.does_line_match(line_pattern, line_output))
+                return false
+            return this.try_match_pattern(pattern_offset + 1, fif_output, offset + 1)
+        }
+        while (offset < fif_output.length) {
+            if (this.try_match_pattern(pattern_offset + 1, fif_output, offset))
+                return true
+            offset = offset + 1
+        }
+        return false
+    }
+
+    static split_line_to_cmd_and_comment(/**string*/ trimmed_line) {
+        const pos = trimmed_line.indexOf("//")
+        if (pos === -1)
+            return [trimmed_line, null]
+        else
+            return [trimmed_line.substring(0, pos).trimEnd(), trimmed_line.substring(pos + 2).trimStart()]
+    }
+
+    static does_line_match(/**string*/ line_pattern, /**string*/ line_output) {
+        const [cmd_pattern, comment_pattern] = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern)
+        const [cmd_output, comment_output] = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.trim())
+        return cmd_pattern === cmd_output && (comment_pattern === null || comment_pattern === comment_output)
+    }
+}
+
+
 class FuncTestFile {
     constructor(/**string*/ func_filename, /**string*/ artifacts_folder) {
+        this.line_idx = 0
         this.func_filename = func_filename
         this.artifacts_folder = artifacts_folder
         this.compilation_should_fail = false
-        /** @type {FuncTestCaseStderrIncludes[]} */
+        /** @type {FuncTestCaseStderr[]} */
         this.stderr_includes = []
         /** @type {FuncTestCaseInputOutput[]} */
         this.input_output = []
+        /** @type {FuncTestCaseFifCodegen[]} */
+        this.fif_codegen = []
     }
 
     parse_input_from_func_file() {
         const lines = fs.readFileSync(this.func_filename, 'utf-8').split(/\r?\n/)
-        let i = 0
+        this.line_idx = 0
-        while (i < lines.length) {
+
-            const line = lines[i]
+        while (this.line_idx < lines.length) {
+            const line = lines[this.line_idx]
             if (line.startsWith('TESTCASE')) {
                 let s = line.split("|").map(p => p.trim())
                 if (s.length !== 4)
@@ -160,9 +265,13 @@ class FuncTestFile {
             } else if (line.startsWith('@compilation_should_fail')) {
                 this.compilation_should_fail = true
             } else if (line.startsWith('@stderr')) {
-                this.stderr_includes.push(new FuncTestCaseStderrIncludes(line.substring(7).trim()))
+                this.stderr_includes.push(new FuncTestCaseStderr(this.parse_string_value(lines), false))
+            } else if (line.startsWith("@fif_codegen_avoid")) {
+                this.fif_codegen.push(new FuncTestCaseFifCodegen(this.parse_string_value(lines), true))
+            } else if (line.startsWith("@fif_codegen")) {
+                this.fif_codegen.push(new FuncTestCaseFifCodegen(this.parse_string_value(lines), false))
             }
-            i++
+            this.line_idx++
         }
 
         if (this.input_output.length === 0 && !this.compilation_should_fail)
@@ -171,6 +280,31 @@ class FuncTestFile {
             throw new ParseInputError("TESTCASE present, but compilation_should_fail")
     }
 
+    /** @return {string[]} */
+    parse_string_value(/**string[]*/ lines) {
+        // a tag must be followed by a space (single-line), e.g. '@stderr some text'
+        // or be a multi-line value, surrounded by """
+        const line = lines[this.line_idx]
+        const pos_sp = line.indexOf(' ')
+        const is_multi_line = lines[this.line_idx + 1] === '"""'
+        const is_single_line = pos_sp !== -1
+        if (!is_single_line && !is_multi_line)
+            throw new ParseInputError(`${line} value is empty (not followed by a string or a multiline """)`)
+        if (is_single_line && is_multi_line)
+            throw new ParseInputError(`${line.substring(0, pos_sp)} value is both single-line and followed by """`)
+
+        if (is_single_line)
+            return [line.substring(pos_sp + 1).trim()]
+
+        this.line_idx += 2
+        let s_multiline = []
+        while (this.line_idx < lines.length && lines[this.line_idx] !== '"""') {
+            s_multiline.push(lines[this.line_idx])
+            this.line_idx = this.line_idx + 1
+        }
+        return s_multiline
+    }
+
     get_compiled_fif_filename() {
         return this.artifacts_folder + "/compiled.fif"
     }
@@ -220,6 +354,12 @@ class FuncTestFile {
 
         for (let i = 0; i < stdout_lines.length; ++i)
             this.input_output[i].check(stdout_lines, i)
+
+        if (this.fif_codegen.length) {
+            const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/)
+            for (let fif_codegen of this.fif_codegen)
+                fif_codegen.check(fif_output)
+        }
     }
 }
 
@@ -244,7 +384,7 @@ async function run_all_tests(/**string[]*/ tests) {
                 print(`  OK, ${testcase.input_output.length} cases`)
         } catch (e) {
             if (e instanceof ParseInputError) {
-                print("  Error parsing input:", e.message)
+                print(`  Error parsing input (cur line #${testcase.line_idx + 1}):`, e.message)
                 process.exit(2)
             } else if (e instanceof FuncCompilationFailedError) {
                 print("  Error compiling func:", e.message)
@@ -263,6 +403,11 @@ async function run_all_tests(/**string[]*/ tests) {
                 print(e.output.trimEnd())
                 print("  Was compiled to:", testcase.get_compiled_fif_filename())
                 process.exit(2)
+            } else if (e instanceof CompareFifCodegenError) {
+                print("  Mismatch in fif codegen:", e.message)
+                print("  Was compiled to:", testcase.get_compiled_fif_filename())
+                print(fs.readFileSync(testcase.get_compiled_fif_filename(), 'utf-8'))
+                process.exit(2)
             }
             throw e
         }

crypto/func/auto-tests/run_tests.py

@@ -84,6 +84,10 @@ class CompareOutputError(Exception):
         self.output = output
 
 
+class CompareFifCodegenError(Exception):
+    pass
+
+
 class FuncTestCaseInputOutput:
     """
     In positive tests, there are several testcases "input X should produce output Y".
@@ -114,33 +118,131 @@ class FuncTestCaseInputOutput:
             raise CompareOutputError("error on case %d: expected '%s', found '%s'" % (line_idx + 1, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines))
 
 
-class FuncTestCaseStderrIncludes:
+class FuncTestCaseStderr:
     """
     @stderr checks, when compilation fails, that stderr (compilation error) is expected.
+    If it's multiline, all lines must be present in specified order.
     """
 
-    def __init__(self, expected_substr: str):
+    def __init__(self, stderr_pattern: list[str], avoid: bool):
-        self.expected_substr = expected_substr
+        self.stderr_pattern = stderr_pattern
+        self.avoid = avoid
 
     def check(self, stderr: str):
-        if self.expected_substr not in stderr:
+        line_match = self.find_pattern_in_stderr(stderr.splitlines())
-            raise CompareOutputError("pattern '%s' not found in stderr" % self.expected_substr, stderr)
+        if line_match == -1 and not self.avoid:
+            raise CompareOutputError("pattern not found in stderr:\n%s" %
+                                     "\n".join(map(lambda x: "    " + x, self.stderr_pattern)), stderr)
+        elif line_match != -1 and self.avoid:
+            raise CompareOutputError("pattern found (line %d), but not expected to be:\n%s" %
+                                     (line_match + 1, "\n".join(map(lambda x: "    " + x, self.stderr_pattern))), stderr)
+
+    def find_pattern_in_stderr(self, stderr: list[str]) -> int:
+        for line_start in range(len(stderr)):
+            if self.try_match_pattern(0, stderr, line_start):
+                return line_start
+        return -1
+
+    def try_match_pattern(self, pattern_offset: int, stderr: list[str], offset: int) -> bool:
+        if pattern_offset >= len(self.stderr_pattern):
+            return True
+        if offset >= len(stderr):
+            return False
+
+        line_pattern = self.stderr_pattern[pattern_offset]
+        line_output = stderr[offset]
+        return line_output.find(line_pattern) != -1 and self.try_match_pattern(pattern_offset + 1, stderr, offset + 1)
+
+
+class FuncTestCaseFifCodegen:
+    """
+    @fif_codegen checks that contents of compiled.fif matches the expected pattern.
+    @fif_codegen_avoid checks that is does not match the pattern.
+    The pattern is a multiline piece of fift code, optionally with "..." meaning "any lines here".
+    See tests/codegen_check_demo.fc of how it looks.
+    A notable thing about indentations (spaces at line starts):
+    Taking them into account will complicate the code without reasonable profit,
+    that's why we just trim every string.
+    And one more word about //comments. FunC inserts them into fift output.
+    If a line in the pattern contains a //comment, it's expected to be equal.
+    If a line does not, we just compare a command.
+    """
+
+    def __init__(self, fif_pattern: list[str], avoid: bool):
+        self.fif_pattern = [s.strip() for s in fif_pattern]
+        self.avoid = avoid
+
+    def check(self, fif_output: list[str]):
+        # in case there are no comments at all (-S not set, or from wasm), drop them from fif_pattern
+        has_comments = any("//" in line and "generated from" not in line for line in fif_output)
+        if not has_comments:
+            self.fif_pattern = [FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(s)[0] for s in self.fif_pattern]
+            self.fif_pattern = [s for s in self.fif_pattern if s != ""]
+
+        line_match = self.find_pattern_in_fif_output(fif_output)
+        if line_match == -1 and not self.avoid:
+            raise CompareFifCodegenError("pattern not found:\n%s" %
+                                         "\n".join(map(lambda x: "    " + x, self.fif_pattern)))
+        elif line_match != -1 and self.avoid:
+            raise CompareFifCodegenError("pattern found (line %d), but not expected to be:\n%s" %
+                                         (line_match + 1, "\n".join(map(lambda x: "    " + x, self.fif_pattern))))
+
+    def find_pattern_in_fif_output(self, fif_output: list[str]) -> int:
+        for line_start in range(len(fif_output)):
+            if self.try_match_pattern(0, fif_output, line_start):
+                return line_start
+        return -1
+
+    def try_match_pattern(self, pattern_offset: int, fif_output: list[str], offset: int) -> bool:
+        if pattern_offset >= len(self.fif_pattern):
+            return True
+        if offset >= len(fif_output):
+            return False
+        line_pattern = self.fif_pattern[pattern_offset]
+        line_output = fif_output[offset]
+
+        if line_pattern != "...":
+            if not FuncTestCaseFifCodegen.does_line_match(line_pattern, line_output):
+                return False
+            return self.try_match_pattern(pattern_offset + 1, fif_output, offset + 1)
+        while offset < len(fif_output):
+            if self.try_match_pattern(pattern_offset + 1, fif_output, offset):
+                return True
+            offset = offset + 1
+        return False
+
+    @staticmethod
+    def split_line_to_cmd_and_comment(trimmed_line: str) -> tuple[str, str | None]:
+        pos = trimmed_line.find("//")
+        if pos == -1:
+            return trimmed_line, None
+        else:
+            return trimmed_line[:pos].rstrip(), trimmed_line[pos + 2:].lstrip()
+
+    @staticmethod
+    def does_line_match(line_pattern: str, line_output: str) -> bool:
+        cmd_pattern, comment_pattern = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern)
+        cmd_output, comment_output = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.strip())
+        return cmd_pattern == cmd_output and (comment_pattern is None or comment_pattern == comment_output)
 
 
 class FuncTestFile:
     def __init__(self, func_filename: str, artifacts_folder: str):
+        self.line_idx = 0
         self.func_filename = func_filename
         self.artifacts_folder = artifacts_folder
         self.compilation_should_fail = False
-        self.stderr_includes: list[FuncTestCaseStderrIncludes] = []
+        self.stderr_includes: list[FuncTestCaseStderr] = []
         self.input_output: list[FuncTestCaseInputOutput] = []
+        self.fif_codegen: list[FuncTestCaseFifCodegen] = []
 
     def parse_input_from_func_file(self):
         with open(self.func_filename, "r") as fd:
             lines = fd.read().splitlines()
-        i = 0
+        self.line_idx = 0
-        while i < len(lines):
+
-            line = lines[i]
+        while self.line_idx < len(lines):
+            line = lines[self.line_idx]
             if line.startswith("TESTCASE"):
                 s = [x.strip() for x in line.split("|")]
                 if len(s) != 4:
@@ -149,18 +251,43 @@ class FuncTestFile:
             elif line.startswith("@compilation_should_fail"):
                 self.compilation_should_fail = True
             elif line.startswith("@stderr"):
-                self.stderr_includes.append(FuncTestCaseStderrIncludes(line[7:].strip()))
+                self.stderr_includes.append(FuncTestCaseStderr(self.parse_string_value(lines), False))
-            i = i + 1
+            elif line.startswith("@fif_codegen_avoid"):
+                self.fif_codegen.append(FuncTestCaseFifCodegen(self.parse_string_value(lines), True))
+            elif line.startswith("@fif_codegen"):
+                self.fif_codegen.append(FuncTestCaseFifCodegen(self.parse_string_value(lines), False))
+            self.line_idx = self.line_idx + 1
 
         if len(self.input_output) == 0 and not self.compilation_should_fail:
             raise ParseInputError("no TESTCASE present")
         if len(self.input_output) != 0 and self.compilation_should_fail:
             raise ParseInputError("TESTCASE present, but compilation_should_fail")
 
+    def parse_string_value(self, lines: list[str]) -> list[str]:
+        # a tag must be followed by a space (single-line), e.g. '@stderr some text'
+        # or be a multi-line value, surrounded by """
+        line = lines[self.line_idx]
+        pos_sp = line.find(' ')
+        is_multi_line = lines[self.line_idx + 1] == '"""'
+        is_single_line = pos_sp != -1
+        if not is_single_line and not is_multi_line:
+            raise ParseInputError('%s value is empty (not followed by a string or a multiline """)' % line)
+        if is_single_line and is_multi_line:
+            raise ParseInputError('%s value is both single-line and followed by """' % line[:pos_sp])
+
+        if is_single_line:
+            return [line[pos_sp + 1:].strip()]
+
+        self.line_idx += 2
+        s_multiline = []
+        while self.line_idx < len(lines) and lines[self.line_idx] != '"""':
+            s_multiline.append(lines[self.line_idx])
+            self.line_idx = self.line_idx + 1
+        return s_multiline
+
     def get_compiled_fif_filename(self):
         return self.artifacts_folder + "/compiled.fif"
 
-    @property
     def get_runner_fif_filename(self):
         return self.artifacts_folder + "/runner.fif"
 
@@ -181,12 +308,12 @@ class FuncTestFile:
         if exit_code != 0 and not self.compilation_should_fail:
             raise FuncCompilationFailedError("func exit_code = %d" % exit_code, stderr)
 
-        with open(self.get_runner_fif_filename, "w") as f:
+        with open(self.get_runner_fif_filename(), "w") as f:
             f.write("\"%s\" include <s constant code\n" % self.get_compiled_fif_filename())
             for t in self.input_output:
                 f.write("%s %d code 1 runvmx abort\"exitcode is not 0\" .s cr { drop } depth 1- times\n" % (t.input, t.method_id))
 
-        res = subprocess.run([FIFT_EXECUTABLE, self.get_runner_fif_filename], capture_output=True, timeout=10)
+        res = subprocess.run([FIFT_EXECUTABLE, self.get_runner_fif_filename()], capture_output=True, timeout=10)
         exit_code = res.returncode
         stderr = str(res.stderr, "utf-8")
         stdout = str(res.stdout, "utf-8")
@@ -202,6 +329,12 @@ class FuncTestFile:
         for i in range(len(stdout_lines)):
             self.input_output[i].check(stdout_lines, i)
 
+        if len(self.fif_codegen):
+            with(open(self.get_compiled_fif_filename()) as fd):
+                fif_output = fd.readlines()
+            for fif_codegen in self.fif_codegen:
+                fif_codegen.check(fif_output)
+
 
 def run_all_tests(tests: list[str]):
     for ti in range(len(tests)):
@@ -222,7 +355,7 @@ def run_all_tests(tests: list[str]):
             else:
                 print("  OK, %d cases" % len(testcase.input_output), file=sys.stderr)
         except ParseInputError as e:
-            print("  Error parsing input:", e, file=sys.stderr)
+            print("  Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr)
             exit(2)
         except FuncCompilationFailedError as e:
             print("  Error compiling func:", e, file=sys.stderr)
@@ -244,6 +377,11 @@ def run_all_tests(tests: list[str]):
             print(e.output.rstrip(), file=sys.stderr)
             print("  Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr)
             exit(2)
+        except CompareFifCodegenError as e:
+            print("  Mismatch in fif codegen:", e, file=sys.stderr)
+            print("  Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr)
+            print(open(testcase.get_compiled_fif_filename()).read(), file=sys.stderr)
+            exit(2)
 
 
 tests = CmdLineOptions(sys.argv).find_tests()

crypto/func/auto-tests/tests/codegen_check_demo.fc

@@ -0,0 +1,95 @@
+int test1() method_id(101) {
+    var x = false;
+    if (x == true) {
+        x = 100500;
+    }
+    return x;
+}
+
+_ main(s) {
+    var (z, t) = (17, s);
+    while (z > 0) {
+        t = s;
+        z -= 1;
+    }
+    return ~ t;
+}
+
+{-
+    method_id | in | out
+TESTCASE | 0   | 1  | -2
+TESTCASE | 0   | 5  | -6
+TESTCASE | 101 |    | 0
+
+Below, I just give examples of @fif_codegen tag:
+* a pattern can be single-line (after the tag), or multi-line, surrounded with """
+* there may be multiple @fif_codegen, they all will be checked
+* identation (spaces) is not checked intentionally
+* "..." means any number of any lines
+* lines not divided with "..." are expected to be consecutive in fif output
+* //comments can be omitted, but if present, they are also expected to be equal
+* there is also a tag @fif_codegen_avoid to check a pattern does not occur
+
+@fif_codegen
+"""
+main PROC:<{
+  // s
+  17 PUSHINT     //  s _3=17
+  OVER           //  s z=17 t
+  WHILE:<{
+    ...
+  }>DO<{ //  s z t
+    ...
+    s1 s(-1) PUXC //  s t z
+    ...
+  2 1 BLKDROP2
+  ...
+}>
+"""
+
+@fif_codegen
+"""
+main PROC:<{
+  ...
+  WHILE:<{
+    ...
+           }>DO<{
+  ...
+    }>
+        }END>c
+"""
+
+@fif_codegen
+"""
+    OVER
+    0 GTINT   //  s z t _5
+"""
+
+@fif_codegen
+"""
+  "Asm.fif" include
+  ...
+  PROGRAM{
+  ...
+  }END>c
+"""
+
+@fif_codegen
+"""
+test1 PROC:<{
+//
+FALSE
+}>
+"""
+
+@fif_codegen NOT //  _8
+@fif_codegen main PROC:<{
+
+@fif_codegen_avoid PROCINLINE
+@fif_codegen_avoid END c
+@fif_codegen_avoid
+"""
+multiline
+can also be
+"""
+-}