external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

[FunC] Enrich testing framework, add fif output patterns

Browse source 
* @fif_codegen to match compiled.fif against an expected pattern
* @fif_codegen_avoid to ensure compiled.fif doesn't contain a substring
* both in Python and JS run_tests
* consider tests/codegen_check_demo.fc for examples
This commit is contained in:
Aleksandr Kirsanov 2024-04-21 23:21:07 +03:00
parent cbd78964c5
commit bac4e3df97
No known key found for this signature in database
GPG key ID: B758BBAA01FFB3D3
3 changed files with 405 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

169
crypto/func/auto-tests/run_tests.js
View file

@ -89,6 +89,9 @@ class CompareOutputError extends Error {
}
}
class CompareFifCodegenError extends Error {
}
/*
* In positive tests, there are several testcases "input X should produce output Y".
@ -124,34 +127,136 @@ class FuncTestCaseInputOutput {
/*
* @stderr checks, when compilation fails, that stderr (compilation error) is expected.
* If it's multiline, all lines must be present in specified order.
*/
class FuncTestCaseStderrIncludes {
constructor(/**string*/ expected_substr) {
this.expected_substr = expected_substr
class FuncTestCaseStderr {
constructor(/**string[]*/ stderr_pattern, /**boolean*/ avoid) {
this.stderr_pattern = stderr_pattern
this.avoid = avoid
}
check(/**string*/ stderr) {
if (!stderr.includes(this.expected_substr))
throw new CompareOutputError(`pattern '${this.expected_substr}' not found in stderr`, stderr)
const line_match = this.find_pattern_in_stderr(stderr.split(/\n/))
if (line_match === -1 && !this.avoid)
throw new CompareOutputError("pattern not found in stderr:\n" +
this.stderr_pattern.map(x => " " + x).join("\n"), stderr)
else if (line_match !== -1 && this.avoid)
throw new CompareOutputError(`pattern found (line ${line_match + 1}), but not expected to be:\n` +
this.stderr_pattern.map(x => " " + x).join("\n"), stderr)
}
find_pattern_in_stderr(/**string[]*/ stderr) {
for (let line_start = 0; line_start < stderr.length; ++line_start)
if (this.try_match_pattern(0, stderr, line_start))
return line_start
return -1
}
try_match_pattern(/**number*/ pattern_offset, /**string[]*/ stderr, /**number*/ offset) {
if (pattern_offset >= this.stderr_pattern.length)
return true
if (offset >= stderr.length)
return false
const line_pattern = this.stderr_pattern[pattern_offset]
const line_output = stderr[offset]
return line_output.includes(line_pattern) && this.try_match_pattern(pattern_offset + 1, stderr, offset + 1)
}
}
/*
* @fif_codegen checks that contents of compiled.fif matches the expected pattern.
* @fif_codegen_avoid checks that is does not match the pattern.
* See comments in run_tests.py.
*/
class FuncTestCaseFifCodegen {
constructor(/**string[]*/ fif_pattern, /**boolean*/ avoid) {
/** @type {string[]} */
this.fif_pattern = fif_pattern.map(s => s.trim())
this.avoid = avoid
}
check(/**string[]*/ fif_output) {
// in case there are no comments at all (typically for wasm), drop them from fif_pattern
const has_comments = fif_output.some(line => line.includes("//") && !line.includes("generated from"))
if (!has_comments) {
this.fif_pattern = this.fif_pattern.map(s => FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(s)[0])
this.fif_pattern = this.fif_pattern.filter(s => s !== '')
}
const line_match = this.find_pattern_in_fif_output(fif_output)
if (line_match === -1 && !this.avoid)
throw new CompareFifCodegenError("pattern not found:\n" +
this.fif_pattern.map(x => " " + x).join("\n"))
else if (line_match !== -1 && this.avoid)
throw new CompareFifCodegenError(`pattern found (line ${line_match + 1}), but not expected to be:\n` +
this.fif_pattern.map(x => " " + x).join("\n"))
}
find_pattern_in_fif_output(/**string[]*/ fif_output) {
for (let line_start = 0; line_start < fif_output.length; ++line_start)
if (this.try_match_pattern(0, fif_output, line_start))
return line_start
return -1
}
try_match_pattern(/**number*/ pattern_offset, /**string[]*/ fif_output, /**number*/ offset) {
if (pattern_offset >= this.fif_pattern.length)
return true
if (offset >= fif_output.length)
return false
const line_pattern = this.fif_pattern[pattern_offset]
const line_output = fif_output[offset]
if (line_pattern !== "...") {
if (!FuncTestCaseFifCodegen.does_line_match(line_pattern, line_output))
return false
return this.try_match_pattern(pattern_offset + 1, fif_output, offset + 1)
}
while (offset < fif_output.length) {
if (this.try_match_pattern(pattern_offset + 1, fif_output, offset))
return true
offset = offset + 1
}
return false
}
static split_line_to_cmd_and_comment(/**string*/ trimmed_line) {
const pos = trimmed_line.indexOf("//")
if (pos === -1)
return [trimmed_line, null]
else
return [trimmed_line.substring(0, pos).trimEnd(), trimmed_line.substring(pos + 2).trimStart()]
}
static does_line_match(/**string*/ line_pattern, /**string*/ line_output) {
const [cmd_pattern, comment_pattern] = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern)
const [cmd_output, comment_output] = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.trim())
return cmd_pattern === cmd_output && (comment_pattern === null || comment_pattern === comment_output)
}
}
class FuncTestFile {
constructor(/**string*/ func_filename, /**string*/ artifacts_folder) {
this.line_idx = 0
this.func_filename = func_filename
this.artifacts_folder = artifacts_folder
this.compilation_should_fail = false
/** @type {FuncTestCaseStderrIncludes[]} */
/** @type {FuncTestCaseStderr[]} */
this.stderr_includes = []
/** @type {FuncTestCaseInputOutput[]} */
this.input_output = []
/** @type {FuncTestCaseFifCodegen[]} */
this.fif_codegen = []
}
parse_input_from_func_file() {
const lines = fs.readFileSync(this.func_filename, 'utf-8').split(/\r?\n/)
let i = 0
while (i < lines.length) {
const line = lines[i]
this.line_idx = 0
while (this.line_idx < lines.length) {
const line = lines[this.line_idx]
if (line.startsWith('TESTCASE')) {
let s = line.split("|").map(p => p.trim())
if (s.length !== 4)
@ -160,9 +265,13 @@ class FuncTestFile {
} else if (line.startsWith('@compilation_should_fail')) {
this.compilation_should_fail = true
} else if (line.startsWith('@stderr')) {
this.stderr_includes.push(new FuncTestCaseStderrIncludes(line.substring(7).trim()))
this.stderr_includes.push(new FuncTestCaseStderr(this.parse_string_value(lines), false))
} else if (line.startsWith("@fif_codegen_avoid")) {
this.fif_codegen.push(new FuncTestCaseFifCodegen(this.parse_string_value(lines), true))
} else if (line.startsWith("@fif_codegen")) {
this.fif_codegen.push(new FuncTestCaseFifCodegen(this.parse_string_value(lines), false))
}
i++
this.line_idx++
}
if (this.input_output.length === 0 && !this.compilation_should_fail)
@ -171,6 +280,31 @@ class FuncTestFile {
throw new ParseInputError("TESTCASE present, but compilation_should_fail")
}
/** @return {string[]} */
parse_string_value(/**string[]*/ lines) {
// a tag must be followed by a space (single-line), e.g. '@stderr some text'
// or be a multi-line value, surrounded by """
const line = lines[this.line_idx]
const pos_sp = line.indexOf(' ')
const is_multi_line = lines[this.line_idx + 1] === '"""'
const is_single_line = pos_sp !== -1
if (!is_single_line && !is_multi_line)
throw new ParseInputError(`${line} value is empty (not followed by a string or a multiline """)`)
if (is_single_line && is_multi_line)
throw new ParseInputError(`${line.substring(0, pos_sp)} value is both single-line and followed by """`)
if (is_single_line)
return [line.substring(pos_sp + 1).trim()]
this.line_idx += 2
let s_multiline = []
while (this.line_idx < lines.length && lines[this.line_idx] !== '"""') {
s_multiline.push(lines[this.line_idx])
this.line_idx = this.line_idx + 1
}
return s_multiline
}
get_compiled_fif_filename() {
return this.artifacts_folder + "/compiled.fif"
}
@ -220,6 +354,12 @@ class FuncTestFile {
for (let i = 0; i < stdout_lines.length; ++i)
this.input_output[i].check(stdout_lines, i)
if (this.fif_codegen.length) {
const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/)
for (let fif_codegen of this.fif_codegen)
fif_codegen.check(fif_output)
}
}
}
@ -244,7 +384,7 @@ async function run_all_tests(/**string[]*/ tests) {
print(` OK, ${testcase.input_output.length} cases`)
} catch (e) {
if (e instanceof ParseInputError) {
print(" Error parsing input:", e.message)
print(` Error parsing input (cur line #${testcase.line_idx + 1}):`, e.message)
process.exit(2)
} else if (e instanceof FuncCompilationFailedError) {
print(" Error compiling func:", e.message)
@ -263,6 +403,11 @@ async function run_all_tests(/**string[]*/ tests) {
print(e.output.trimEnd())
print(" Was compiled to:", testcase.get_compiled_fif_filename())
process.exit(2)
} else if (e instanceof CompareFifCodegenError) {
print(" Mismatch in fif codegen:", e.message)
print(" Was compiled to:", testcase.get_compiled_fif_filename())
print(fs.readFileSync(testcase.get_compiled_fif_filename(), 'utf-8'))
process.exit(2)
}
throw e
}

168
crypto/func/auto-tests/run_tests.py
View file

@ -84,6 +84,10 @@ class CompareOutputError(Exception):
self.output = output
class CompareFifCodegenError(Exception):
pass
class FuncTestCaseInputOutput:
"""
In positive tests, there are several testcases "input X should produce output Y".
@ -114,33 +118,131 @@ class FuncTestCaseInputOutput:
raise CompareOutputError("error on case %d: expected '%s', found '%s'" % (line_idx + 1, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines))
class FuncTestCaseStderrIncludes:
class FuncTestCaseStderr:
"""
@stderr checks, when compilation fails, that stderr (compilation error) is expected.
If it's multiline, all lines must be present in specified order.
"""
def __init__(self, expected_substr: str):
self.expected_substr = expected_substr
def __init__(self, stderr_pattern: list[str], avoid: bool):
self.stderr_pattern = stderr_pattern
self.avoid = avoid
def check(self, stderr: str):
if self.expected_substr not in stderr:
raise CompareOutputError("pattern '%s' not found in stderr" % self.expected_substr, stderr)
line_match = self.find_pattern_in_stderr(stderr.splitlines())
if line_match == -1 and not self.avoid:
raise CompareOutputError("pattern not found in stderr:\n%s" %
"\n".join(map(lambda x: " " + x, self.stderr_pattern)), stderr)
elif line_match != -1 and self.avoid:
raise CompareOutputError("pattern found (line %d), but not expected to be:\n%s" %
(line_match + 1, "\n".join(map(lambda x: " " + x, self.stderr_pattern))), stderr)
def find_pattern_in_stderr(self, stderr: list[str]) -> int:
for line_start in range(len(stderr)):
if self.try_match_pattern(0, stderr, line_start):
return line_start
return -1
def try_match_pattern(self, pattern_offset: int, stderr: list[str], offset: int) -> bool:
if pattern_offset >= len(self.stderr_pattern):
return True
if offset >= len(stderr):
return False
line_pattern = self.stderr_pattern[pattern_offset]
line_output = stderr[offset]
return line_output.find(line_pattern) != -1 and self.try_match_pattern(pattern_offset + 1, stderr, offset + 1)
class FuncTestCaseFifCodegen:
"""
@fif_codegen checks that contents of compiled.fif matches the expected pattern.
@fif_codegen_avoid checks that is does not match the pattern.
The pattern is a multiline piece of fift code, optionally with "..." meaning "any lines here".
See tests/codegen_check_demo.fc of how it looks.
A notable thing about indentations (spaces at line starts):
Taking them into account will complicate the code without reasonable profit,
that's why we just trim every string.
And one more word about //comments. FunC inserts them into fift output.
If a line in the pattern contains a //comment, it's expected to be equal.
If a line does not, we just compare a command.
"""
def __init__(self, fif_pattern: list[str], avoid: bool):
self.fif_pattern = [s.strip() for s in fif_pattern]
self.avoid = avoid
def check(self, fif_output: list[str]):
# in case there are no comments at all (-S not set, or from wasm), drop them from fif_pattern
has_comments = any("//" in line and "generated from" not in line for line in fif_output)
if not has_comments:
self.fif_pattern = [FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(s)[0] for s in self.fif_pattern]
self.fif_pattern = [s for s in self.fif_pattern if s != ""]
line_match = self.find_pattern_in_fif_output(fif_output)
if line_match == -1 and not self.avoid:
raise CompareFifCodegenError("pattern not found:\n%s" %
"\n".join(map(lambda x: " " + x, self.fif_pattern)))
elif line_match != -1 and self.avoid:
raise CompareFifCodegenError("pattern found (line %d), but not expected to be:\n%s" %
(line_match + 1, "\n".join(map(lambda x: " " + x, self.fif_pattern))))
def find_pattern_in_fif_output(self, fif_output: list[str]) -> int:
for line_start in range(len(fif_output)):
if self.try_match_pattern(0, fif_output, line_start):
return line_start
return -1
def try_match_pattern(self, pattern_offset: int, fif_output: list[str], offset: int) -> bool:
if pattern_offset >= len(self.fif_pattern):
return True
if offset >= len(fif_output):
return False
line_pattern = self.fif_pattern[pattern_offset]
line_output = fif_output[offset]
if line_pattern != "...":
if not FuncTestCaseFifCodegen.does_line_match(line_pattern, line_output):
return False
return self.try_match_pattern(pattern_offset + 1, fif_output, offset + 1)
while offset < len(fif_output):
if self.try_match_pattern(pattern_offset + 1, fif_output, offset):
return True
offset = offset + 1
return False
@staticmethod
def split_line_to_cmd_and_comment(trimmed_line: str) -> tuple[str, str | None]:
pos = trimmed_line.find("//")
if pos == -1:
return trimmed_line, None
else:
return trimmed_line[:pos].rstrip(), trimmed_line[pos + 2:].lstrip()
@staticmethod
def does_line_match(line_pattern: str, line_output: str) -> bool:
cmd_pattern, comment_pattern = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern)
cmd_output, comment_output = FuncTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.strip())
return cmd_pattern == cmd_output and (comment_pattern is None or comment_pattern == comment_output)
class FuncTestFile:
def __init__(self, func_filename: str, artifacts_folder: str):
self.line_idx = 0
self.func_filename = func_filename
self.artifacts_folder = artifacts_folder
self.compilation_should_fail = False
self.stderr_includes: list[FuncTestCaseStderrIncludes] = []
self.stderr_includes: list[FuncTestCaseStderr] = []
self.input_output: list[FuncTestCaseInputOutput] = []
self.fif_codegen: list[FuncTestCaseFifCodegen] = []
def parse_input_from_func_file(self):
with open(self.func_filename, "r") as fd:
lines = fd.read().splitlines()
i = 0
while i < len(lines):
line = lines[i]
self.line_idx = 0
while self.line_idx < len(lines):
line = lines[self.line_idx]
if line.startswith("TESTCASE"):
s = [x.strip() for x in line.split("|")]
if len(s) != 4:
@ -149,18 +251,43 @@ class FuncTestFile:
elif line.startswith("@compilation_should_fail"):
self.compilation_should_fail = True
elif line.startswith("@stderr"):
self.stderr_includes.append(FuncTestCaseStderrIncludes(line[7:].strip()))
i = i + 1
self.stderr_includes.append(FuncTestCaseStderr(self.parse_string_value(lines), False))
elif line.startswith("@fif_codegen_avoid"):
self.fif_codegen.append(FuncTestCaseFifCodegen(self.parse_string_value(lines), True))
elif line.startswith("@fif_codegen"):
self.fif_codegen.append(FuncTestCaseFifCodegen(self.parse_string_value(lines), False))
self.line_idx = self.line_idx + 1
if len(self.input_output) == 0 and not self.compilation_should_fail:
raise ParseInputError("no TESTCASE present")
if len(self.input_output) != 0 and self.compilation_should_fail:
raise ParseInputError("TESTCASE present, but compilation_should_fail")
def parse_string_value(self, lines: list[str]) -> list[str]:
# a tag must be followed by a space (single-line), e.g. '@stderr some text'
# or be a multi-line value, surrounded by """
line = lines[self.line_idx]
pos_sp = line.find(' ')
is_multi_line = lines[self.line_idx + 1] == '"""'
is_single_line = pos_sp != -1
if not is_single_line and not is_multi_line:
raise ParseInputError('%s value is empty (not followed by a string or a multiline """)' % line)
if is_single_line and is_multi_line:
raise ParseInputError('%s value is both single-line and followed by """' % line[:pos_sp])
if is_single_line:
return [line[pos_sp + 1:].strip()]
self.line_idx += 2
s_multiline = []
while self.line_idx < len(lines) and lines[self.line_idx] != '"""':
s_multiline.append(lines[self.line_idx])
self.line_idx = self.line_idx + 1
return s_multiline
def get_compiled_fif_filename(self):
return self.artifacts_folder + "/compiled.fif"
@property
def get_runner_fif_filename(self):
return self.artifacts_folder + "/runner.fif"
@ -181,12 +308,12 @@ class FuncTestFile:
if exit_code != 0 and not self.compilation_should_fail:
raise FuncCompilationFailedError("func exit_code = %d" % exit_code, stderr)
with open(self.get_runner_fif_filename, "w") as f:
with open(self.get_runner_fif_filename(), "w") as f:
f.write("\"%s\" include <s constant code\n" % self.get_compiled_fif_filename())
for t in self.input_output:
f.write("%s %d code 1 runvmx abort\"exitcode is not 0\" .s cr { drop } depth 1- times\n" % (t.input, t.method_id))
res = subprocess.run([FIFT_EXECUTABLE, self.get_runner_fif_filename], capture_output=True, timeout=10)
res = subprocess.run([FIFT_EXECUTABLE, self.get_runner_fif_filename()], capture_output=True, timeout=10)
exit_code = res.returncode
stderr = str(res.stderr, "utf-8")
stdout = str(res.stdout, "utf-8")
@ -202,6 +329,12 @@ class FuncTestFile:
for i in range(len(stdout_lines)):
self.input_output[i].check(stdout_lines, i)
if len(self.fif_codegen):
with(open(self.get_compiled_fif_filename()) as fd):
fif_output = fd.readlines()
for fif_codegen in self.fif_codegen:
fif_codegen.check(fif_output)
def run_all_tests(tests: list[str]):
for ti in range(len(tests)):
@ -222,7 +355,7 @@ def run_all_tests(tests: list[str]):
else:
print(" OK, %d cases" % len(testcase.input_output), file=sys.stderr)
except ParseInputError as e:
print(" Error parsing input:", e, file=sys.stderr)
print(" Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr)
exit(2)
except FuncCompilationFailedError as e:
print(" Error compiling func:", e, file=sys.stderr)
@ -244,6 +377,11 @@ def run_all_tests(tests: list[str]):
print(e.output.rstrip(), file=sys.stderr)
print(" Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr)
exit(2)
except CompareFifCodegenError as e:
print(" Mismatch in fif codegen:", e, file=sys.stderr)
print(" Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr)
print(open(testcase.get_compiled_fif_filename()).read(), file=sys.stderr)
exit(2)
tests = CmdLineOptions(sys.argv).find_tests()

95
crypto/func/auto-tests/tests/codegen_check_demo.fc Normal file
View file

@ -0,0 +1,95 @@
int test1() method_id(101) {
var x = false;
if (x == true) {
x = 100500;
}
return x;
}
_ main(s) {
var (z, t) = (17, s);
while (z > 0) {
t = s;
z -= 1;
}
return ~ t;
}
{-
method_id | in | out
TESTCASE | 0 | 1 | -2
TESTCASE | 0 | 5 | -6
TESTCASE | 101 | | 0
Below, I just give examples of @fif_codegen tag:
* a pattern can be single-line (after the tag), or multi-line, surrounded with """
* there may be multiple @fif_codegen, they all will be checked
* identation (spaces) is not checked intentionally
* "..." means any number of any lines
* lines not divided with "..." are expected to be consecutive in fif output
* //comments can be omitted, but if present, they are also expected to be equal
* there is also a tag @fif_codegen_avoid to check a pattern does not occur
@fif_codegen
"""
main PROC:<{
// s
17 PUSHINT // s _3=17
OVER // s z=17 t
WHILE:<{
...
}>DO<{ // s z t
...
s1 s(-1) PUXC // s t z
...
2 1 BLKDROP2
...
}>
"""
@fif_codegen
"""
main PROC:<{
...
WHILE:<{
...
}>DO<{
...
}>
}END>c
"""
@fif_codegen
"""
OVER
0 GTINT // s z t _5
"""
@fif_codegen
"""
"Asm.fif" include
...
PROGRAM{
...
}END>c
"""
@fif_codegen
"""
test1 PROC:<{
//
FALSE
}>
"""
@fif_codegen NOT // _8
@fif_codegen main PROC:<{
@fif_codegen_avoid PROCINLINE
@fif_codegen_avoid END c
@fif_codegen_avoid
"""
multiline
can also be
"""
-}