diff --git a/crypto/func/auto-tests/run_tests.js b/crypto/func/auto-tests/run_tests.js index 19db5ffe..28a0d536 100644 --- a/crypto/func/auto-tests/run_tests.js +++ b/crypto/func/auto-tests/run_tests.js @@ -89,6 +89,9 @@ class CompareOutputError extends Error { } } +class CompareFifCodegenError extends Error { +} + /* * In positive tests, there are several testcases "input X should produce output Y". @@ -124,34 +127,136 @@ class FuncTestCaseInputOutput { /* * @stderr checks, when compilation fails, that stderr (compilation error) is expected. + * If it's multiline, all lines must be present in specified order. */ -class FuncTestCaseStderrIncludes { - constructor(/**string*/ expected_substr) { - this.expected_substr = expected_substr +class FuncTestCaseStderr { + constructor(/**string[]*/ stderr_pattern, /**boolean*/ avoid) { + this.stderr_pattern = stderr_pattern + this.avoid = avoid } check(/**string*/ stderr) { - if (!stderr.includes(this.expected_substr)) - throw new CompareOutputError(`pattern '${this.expected_substr}' not found in stderr`, stderr) + const line_match = this.find_pattern_in_stderr(stderr.split(/\n/)) + if (line_match === -1 && !this.avoid) + throw new CompareOutputError("pattern not found in stderr:\n" + + this.stderr_pattern.map(x => " " + x).join("\n"), stderr) + else if (line_match !== -1 && this.avoid) + throw new CompareOutputError(`pattern found (line ${line_match + 1}), but not expected to be:\n` + + this.stderr_pattern.map(x => " " + x).join("\n"), stderr) + } + + find_pattern_in_stderr(/**string[]*/ stderr) { + for (let line_start = 0; line_start < stderr.length; ++line_start) + if (this.try_match_pattern(0, stderr, line_start)) + return line_start + return -1 + } + + try_match_pattern(/**number*/ pattern_offset, /**string[]*/ stderr, /**number*/ offset) { + if (pattern_offset >= this.stderr_pattern.length) + return true + if (offset >= stderr.length) + return false + + const line_pattern = this.stderr_pattern[pattern_offset] + const line_output = stderr[offset] + return line_output.includes(line_pattern) && this.try_match_pattern(pattern_offset + 1, stderr, offset + 1) } } +/* + * @fif_codegen checks that contents of compiled.fif matches the expected pattern. + * @fif_codegen_avoid checks that is does not match the pattern. + * See comments in run_tests.py. + */ +class FuncTestCaseFifCodegen { + constructor(/**string[]*/ fif_pattern, /**boolean*/ avoid) { + /** @type {string[]} */ + this.fif_pattern = fif_pattern.map(s => s.trim()) + this.avoid = avoid + } + + check(/**string[]*/ fif_output) { + // in case there are no comments at all (typically for wasm), drop them from fif_pattern + const has_comments = fif_output.some(line => line.includes("//") && !line.includes("generated from")) + if (!has_comments) { + this.fif_pattern = this.fif_pattern.map(s => FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(s)[0]) + this.fif_pattern = this.fif_pattern.filter(s => s !== '') + } + + const line_match = this.find_pattern_in_fif_output(fif_output) + if (line_match === -1 && !this.avoid) + throw new CompareFifCodegenError("pattern not found:\n" + + this.fif_pattern.map(x => " " + x).join("\n")) + else if (line_match !== -1 && this.avoid) + throw new CompareFifCodegenError(`pattern found (line ${line_match + 1}), but not expected to be:\n` + + this.fif_pattern.map(x => " " + x).join("\n")) + } + + find_pattern_in_fif_output(/**string[]*/ fif_output) { + for (let line_start = 0; line_start < fif_output.length; ++line_start) + if (this.try_match_pattern(0, fif_output, line_start)) + return line_start + return -1 + } + + try_match_pattern(/**number*/ pattern_offset, /**string[]*/ fif_output, /**number*/ offset) { + if (pattern_offset >= this.fif_pattern.length) + return true + if (offset >= fif_output.length) + return false + const line_pattern = this.fif_pattern[pattern_offset] + const line_output = fif_output[offset] + + if (line_pattern !== "...") { + if (!FuncTestCaseFifCodegen.does_line_match(line_pattern, line_output)) + return false + return this.try_match_pattern(pattern_offset + 1, fif_output, offset + 1) + } + while (offset < fif_output.length) { + if (this.try_match_pattern(pattern_offset + 1, fif_output, offset)) + return true + offset = offset + 1 + } + return false + } + + static split_line_to_cmd_and_comment(/**string*/ trimmed_line) { + const pos = trimmed_line.indexOf("//") + if (pos === -1) + return [trimmed_line, null] + else + return [trimmed_line.substring(0, pos).trimEnd(), trimmed_line.substring(pos + 2).trimStart()] + } + + static does_line_match(/**string*/ line_pattern, /**string*/ line_output) { + const [cmd_pattern, comment_pattern] = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern) + const [cmd_output, comment_output] = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.trim()) + return cmd_pattern === cmd_output && (comment_pattern === null || comment_pattern === comment_output) + } +} + + class FuncTestFile { constructor(/**string*/ func_filename, /**string*/ artifacts_folder) { + this.line_idx = 0 this.func_filename = func_filename this.artifacts_folder = artifacts_folder this.compilation_should_fail = false - /** @type {FuncTestCaseStderrIncludes[]} */ + /** @type {FuncTestCaseStderr[]} */ this.stderr_includes = [] /** @type {FuncTestCaseInputOutput[]} */ this.input_output = [] + /** @type {FuncTestCaseFifCodegen[]} */ + this.fif_codegen = [] } parse_input_from_func_file() { const lines = fs.readFileSync(this.func_filename, 'utf-8').split(/\r?\n/) - let i = 0 - while (i < lines.length) { - const line = lines[i] + this.line_idx = 0 + + while (this.line_idx < lines.length) { + const line = lines[this.line_idx] if (line.startsWith('TESTCASE')) { let s = line.split("|").map(p => p.trim()) if (s.length !== 4) @@ -160,9 +265,13 @@ class FuncTestFile { } else if (line.startsWith('@compilation_should_fail')) { this.compilation_should_fail = true } else if (line.startsWith('@stderr')) { - this.stderr_includes.push(new FuncTestCaseStderrIncludes(line.substring(7).trim())) + this.stderr_includes.push(new FuncTestCaseStderr(this.parse_string_value(lines), false)) + } else if (line.startsWith("@fif_codegen_avoid")) { + this.fif_codegen.push(new FuncTestCaseFifCodegen(this.parse_string_value(lines), true)) + } else if (line.startsWith("@fif_codegen")) { + this.fif_codegen.push(new FuncTestCaseFifCodegen(this.parse_string_value(lines), false)) } - i++ + this.line_idx++ } if (this.input_output.length === 0 && !this.compilation_should_fail) @@ -171,6 +280,31 @@ class FuncTestFile { throw new ParseInputError("TESTCASE present, but compilation_should_fail") } + /** @return {string[]} */ + parse_string_value(/**string[]*/ lines) { + // a tag must be followed by a space (single-line), e.g. '@stderr some text' + // or be a multi-line value, surrounded by """ + const line = lines[this.line_idx] + const pos_sp = line.indexOf(' ') + const is_multi_line = lines[this.line_idx + 1] === '"""' + const is_single_line = pos_sp !== -1 + if (!is_single_line && !is_multi_line) + throw new ParseInputError(`${line} value is empty (not followed by a string or a multiline """)`) + if (is_single_line && is_multi_line) + throw new ParseInputError(`${line.substring(0, pos_sp)} value is both single-line and followed by """`) + + if (is_single_line) + return [line.substring(pos_sp + 1).trim()] + + this.line_idx += 2 + let s_multiline = [] + while (this.line_idx < lines.length && lines[this.line_idx] !== '"""') { + s_multiline.push(lines[this.line_idx]) + this.line_idx = this.line_idx + 1 + } + return s_multiline + } + get_compiled_fif_filename() { return this.artifacts_folder + "/compiled.fif" } @@ -220,6 +354,12 @@ class FuncTestFile { for (let i = 0; i < stdout_lines.length; ++i) this.input_output[i].check(stdout_lines, i) + + if (this.fif_codegen.length) { + const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/) + for (let fif_codegen of this.fif_codegen) + fif_codegen.check(fif_output) + } } } @@ -244,7 +384,7 @@ async function run_all_tests(/**string[]*/ tests) { print(` OK, ${testcase.input_output.length} cases`) } catch (e) { if (e instanceof ParseInputError) { - print(" Error parsing input:", e.message) + print(` Error parsing input (cur line #${testcase.line_idx + 1}):`, e.message) process.exit(2) } else if (e instanceof FuncCompilationFailedError) { print(" Error compiling func:", e.message) @@ -263,6 +403,11 @@ async function run_all_tests(/**string[]*/ tests) { print(e.output.trimEnd()) print(" Was compiled to:", testcase.get_compiled_fif_filename()) process.exit(2) + } else if (e instanceof CompareFifCodegenError) { + print(" Mismatch in fif codegen:", e.message) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + print(fs.readFileSync(testcase.get_compiled_fif_filename(), 'utf-8')) + process.exit(2) } throw e } diff --git a/crypto/func/auto-tests/run_tests.py b/crypto/func/auto-tests/run_tests.py index 990e5046..3e2cb8a6 100644 --- a/crypto/func/auto-tests/run_tests.py +++ b/crypto/func/auto-tests/run_tests.py @@ -84,6 +84,10 @@ class CompareOutputError(Exception): self.output = output +class CompareFifCodegenError(Exception): + pass + + class FuncTestCaseInputOutput: """ In positive tests, there are several testcases "input X should produce output Y". @@ -114,33 +118,131 @@ class FuncTestCaseInputOutput: raise CompareOutputError("error on case %d: expected '%s', found '%s'" % (line_idx + 1, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) -class FuncTestCaseStderrIncludes: +class FuncTestCaseStderr: """ @stderr checks, when compilation fails, that stderr (compilation error) is expected. + If it's multiline, all lines must be present in specified order. """ - def __init__(self, expected_substr: str): - self.expected_substr = expected_substr + def __init__(self, stderr_pattern: list[str], avoid: bool): + self.stderr_pattern = stderr_pattern + self.avoid = avoid def check(self, stderr: str): - if self.expected_substr not in stderr: - raise CompareOutputError("pattern '%s' not found in stderr" % self.expected_substr, stderr) + line_match = self.find_pattern_in_stderr(stderr.splitlines()) + if line_match == -1 and not self.avoid: + raise CompareOutputError("pattern not found in stderr:\n%s" % + "\n".join(map(lambda x: " " + x, self.stderr_pattern)), stderr) + elif line_match != -1 and self.avoid: + raise CompareOutputError("pattern found (line %d), but not expected to be:\n%s" % + (line_match + 1, "\n".join(map(lambda x: " " + x, self.stderr_pattern))), stderr) + + def find_pattern_in_stderr(self, stderr: list[str]) -> int: + for line_start in range(len(stderr)): + if self.try_match_pattern(0, stderr, line_start): + return line_start + return -1 + + def try_match_pattern(self, pattern_offset: int, stderr: list[str], offset: int) -> bool: + if pattern_offset >= len(self.stderr_pattern): + return True + if offset >= len(stderr): + return False + + line_pattern = self.stderr_pattern[pattern_offset] + line_output = stderr[offset] + return line_output.find(line_pattern) != -1 and self.try_match_pattern(pattern_offset + 1, stderr, offset + 1) + + +class FuncTestCaseFifCodegen: + """ + @fif_codegen checks that contents of compiled.fif matches the expected pattern. + @fif_codegen_avoid checks that is does not match the pattern. + The pattern is a multiline piece of fift code, optionally with "..." meaning "any lines here". + See tests/codegen_check_demo.fc of how it looks. + A notable thing about indentations (spaces at line starts): + Taking them into account will complicate the code without reasonable profit, + that's why we just trim every string. + And one more word about //comments. FunC inserts them into fift output. + If a line in the pattern contains a //comment, it's expected to be equal. + If a line does not, we just compare a command. + """ + + def __init__(self, fif_pattern: list[str], avoid: bool): + self.fif_pattern = [s.strip() for s in fif_pattern] + self.avoid = avoid + + def check(self, fif_output: list[str]): + # in case there are no comments at all (-S not set, or from wasm), drop them from fif_pattern + has_comments = any("//" in line and "generated from" not in line for line in fif_output) + if not has_comments: + self.fif_pattern = [FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(s)[0] for s in self.fif_pattern] + self.fif_pattern = [s for s in self.fif_pattern if s != ""] + + line_match = self.find_pattern_in_fif_output(fif_output) + if line_match == -1 and not self.avoid: + raise CompareFifCodegenError("pattern not found:\n%s" % + "\n".join(map(lambda x: " " + x, self.fif_pattern))) + elif line_match != -1 and self.avoid: + raise CompareFifCodegenError("pattern found (line %d), but not expected to be:\n%s" % + (line_match + 1, "\n".join(map(lambda x: " " + x, self.fif_pattern)))) + + def find_pattern_in_fif_output(self, fif_output: list[str]) -> int: + for line_start in range(len(fif_output)): + if self.try_match_pattern(0, fif_output, line_start): + return line_start + return -1 + + def try_match_pattern(self, pattern_offset: int, fif_output: list[str], offset: int) -> bool: + if pattern_offset >= len(self.fif_pattern): + return True + if offset >= len(fif_output): + return False + line_pattern = self.fif_pattern[pattern_offset] + line_output = fif_output[offset] + + if line_pattern != "...": + if not FuncTestCaseFifCodegen.does_line_match(line_pattern, line_output): + return False + return self.try_match_pattern(pattern_offset + 1, fif_output, offset + 1) + while offset < len(fif_output): + if self.try_match_pattern(pattern_offset + 1, fif_output, offset): + return True + offset = offset + 1 + return False + + @staticmethod + def split_line_to_cmd_and_comment(trimmed_line: str) -> tuple[str, str | None]: + pos = trimmed_line.find("//") + if pos == -1: + return trimmed_line, None + else: + return trimmed_line[:pos].rstrip(), trimmed_line[pos + 2:].lstrip() + + @staticmethod + def does_line_match(line_pattern: str, line_output: str) -> bool: + cmd_pattern, comment_pattern = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern) + cmd_output, comment_output = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.strip()) + return cmd_pattern == cmd_output and (comment_pattern is None or comment_pattern == comment_output) class FuncTestFile: def __init__(self, func_filename: str, artifacts_folder: str): + self.line_idx = 0 self.func_filename = func_filename self.artifacts_folder = artifacts_folder self.compilation_should_fail = False - self.stderr_includes: list[FuncTestCaseStderrIncludes] = [] + self.stderr_includes: list[FuncTestCaseStderr] = [] self.input_output: list[FuncTestCaseInputOutput] = [] + self.fif_codegen: list[FuncTestCaseFifCodegen] = [] def parse_input_from_func_file(self): with open(self.func_filename, "r") as fd: lines = fd.read().splitlines() - i = 0 - while i < len(lines): - line = lines[i] + self.line_idx = 0 + + while self.line_idx < len(lines): + line = lines[self.line_idx] if line.startswith("TESTCASE"): s = [x.strip() for x in line.split("|")] if len(s) != 4: @@ -149,18 +251,43 @@ class FuncTestFile: elif line.startswith("@compilation_should_fail"): self.compilation_should_fail = True elif line.startswith("@stderr"): - self.stderr_includes.append(FuncTestCaseStderrIncludes(line[7:].strip())) - i = i + 1 + self.stderr_includes.append(FuncTestCaseStderr(self.parse_string_value(lines), False)) + elif line.startswith("@fif_codegen_avoid"): + self.fif_codegen.append(FuncTestCaseFifCodegen(self.parse_string_value(lines), True)) + elif line.startswith("@fif_codegen"): + self.fif_codegen.append(FuncTestCaseFifCodegen(self.parse_string_value(lines), False)) + self.line_idx = self.line_idx + 1 if len(self.input_output) == 0 and not self.compilation_should_fail: raise ParseInputError("no TESTCASE present") if len(self.input_output) != 0 and self.compilation_should_fail: raise ParseInputError("TESTCASE present, but compilation_should_fail") + def parse_string_value(self, lines: list[str]) -> list[str]: + # a tag must be followed by a space (single-line), e.g. '@stderr some text' + # or be a multi-line value, surrounded by """ + line = lines[self.line_idx] + pos_sp = line.find(' ') + is_multi_line = lines[self.line_idx + 1] == '"""' + is_single_line = pos_sp != -1 + if not is_single_line and not is_multi_line: + raise ParseInputError('%s value is empty (not followed by a string or a multiline """)' % line) + if is_single_line and is_multi_line: + raise ParseInputError('%s value is both single-line and followed by """' % line[:pos_sp]) + + if is_single_line: + return [line[pos_sp + 1:].strip()] + + self.line_idx += 2 + s_multiline = [] + while self.line_idx < len(lines) and lines[self.line_idx] != '"""': + s_multiline.append(lines[self.line_idx]) + self.line_idx = self.line_idx + 1 + return s_multiline + def get_compiled_fif_filename(self): return self.artifacts_folder + "/compiled.fif" - @property def get_runner_fif_filename(self): return self.artifacts_folder + "/runner.fif" @@ -181,12 +308,12 @@ class FuncTestFile: if exit_code != 0 and not self.compilation_should_fail: raise FuncCompilationFailedError("func exit_code = %d" % exit_code, stderr) - with open(self.get_runner_fif_filename, "w") as f: + with open(self.get_runner_fif_filename(), "w") as f: f.write("\"%s\" include 0) { + t = s; + z -= 1; + } + return ~ t; +} + +{- + method_id | in | out +TESTCASE | 0 | 1 | -2 +TESTCASE | 0 | 5 | -6 +TESTCASE | 101 | | 0 + +Below, I just give examples of @fif_codegen tag: +* a pattern can be single-line (after the tag), or multi-line, surrounded with """ +* there may be multiple @fif_codegen, they all will be checked +* identation (spaces) is not checked intentionally +* "..." means any number of any lines +* lines not divided with "..." are expected to be consecutive in fif output +* //comments can be omitted, but if present, they are also expected to be equal +* there is also a tag @fif_codegen_avoid to check a pattern does not occur + +@fif_codegen +""" +main PROC:<{ + // s + 17 PUSHINT // s _3=17 + OVER // s z=17 t + WHILE:<{ + ... + }>DO<{ // s z t + ... + s1 s(-1) PUXC // s t z + ... + 2 1 BLKDROP2 + ... +}> +""" + +@fif_codegen +""" +main PROC:<{ + ... + WHILE:<{ + ... + }>DO<{ + ... + }> + }END>c +""" + +@fif_codegen +""" + OVER + 0 GTINT // s z t _5 +""" + +@fif_codegen +""" + "Asm.fif" include + ... + PROGRAM{ + ... + }END>c +""" + +@fif_codegen +""" +test1 PROC:<{ +// +FALSE +}> +""" + +@fif_codegen NOT // _8 +@fif_codegen main PROC:<{ + +@fif_codegen_avoid PROCINLINE +@fif_codegen_avoid END c +@fif_codegen_avoid +""" +multiline +can also be +""" +-}