From c73796878a1755c2e09ca05cd4afec13cdbf3620 Mon Sep 17 00:00:00 2001 From: "Ross A. Wollman" Date: Tue, 11 Aug 2020 20:50:11 -0700 Subject: [PATCH 1/2] Add downstream test discrepency reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example Run and Output: ``` $ python tests/report_downstream_test_diffs.py --playwright-js-path=../playwright --api=async MISSING, MISPELLED, OR MISNAMED: async/test_accessibility.py::test_autocomplete … async/test_workers.py::test_should_have_JSHandles_for_console_logs async/test_workers.py::test_should_report_console_logs async/test_workers.py::test_should_report_errors async/test_workers.py::test_should_report_network_activity async/test_workers.py::test_should_report_network_activity_on_worker_creation Missing: 1213, Found: 105 ``` Relates #138. --- tests/report_downstream_test_diffs.py | 85 +++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/report_downstream_test_diffs.py diff --git a/tests/report_downstream_test_diffs.py b/tests/report_downstream_test_diffs.py new file mode 100644 index 000000000..964bfad3f --- /dev/null +++ b/tests/report_downstream_test_diffs.py @@ -0,0 +1,85 @@ +import argparse +import json +import os +import re +import subprocess +import tempfile +from collections import namedtuple + +TestCase = namedtuple("TestCase", ["api", "file", "test"]) + + +def pytest_test_cases(): + p = subprocess.run( + ["pytest", "--browser", "chromium", "--collect-only", "-q"], + stdout=subprocess.PIPE, + ) + regex = ( + r"tests/(?Pa?sync)/test_(?P.*)\.py::test_(?P.*)\[chromium\]" + ) + matches = re.finditer(regex, p.stdout.decode(), re.MULTILINE) + for match in matches: + yield TestCase( + match.group("api"), match.group("file"), match.group("test"), + ) + + +def jest_test_cases(playwright_js_path): + env = os.environ.copy() + env["REPORT_ONLY"] = "true" + with tempfile.NamedTemporaryFile() as results: + subprocess.run( + ["npx", "jest", "--json", "--outputFile", results.name], + env=env, + cwd=playwright_js_path, + stderr=subprocess.DEVNULL, + ) + + for test_suite in json.load(results)["testResults"]: + regex = r"(.*/)?(?P[^/]+)\.spec\.[jt]s$" + file = re.match(regex, test_suite["name"]).group("file") + for assertion in test_suite["assertionResults"]: + yield TestCase("sync", normalized(file), normalized(assertion["title"])) + yield TestCase( + "async", normalized(file), normalized(assertion["title"]) + ) + + +def normalized(original): + return re.sub(r"[^a-z_]", "_", original, flags=re.IGNORECASE) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--playwright-js-path", + type=str, + help="path to playwright JavaScript directory", + required=True, + ) + parser.add_argument( + "--api", + type=str, + help="filter test cases based on API", + choices=["sync", "async"], + ) + args = parser.parse_args() + + python_tests = set(pytest_test_cases()) + javascript_tests = set(jest_test_cases(args.playwright_js_path)) + + if args.api: + javascript_tests = set([x for x in javascript_tests if x[0] == args.api]) + + missing = javascript_tests.difference(python_tests) + found = javascript_tests.intersection(python_tests) + + print("MISSING, MISPELLED, OR MISNAMED:") + for (api, file, test) in sorted(missing): + print(f"{api}/test_{file}.py::test_{test}") + + print(f"\nMissing: {len(missing)}, Found: {len(found)}") + + +if __name__ == "__main__": + main() From b50e46cbd27b565cb96d507393da3f624292ce1e Mon Sep 17 00:00:00 2001 From: "Ross A. Wollman" Date: Thu, 13 Aug 2020 16:20:04 -0700 Subject: [PATCH 2/2] test: accomodate new test runner Also update normalization to be friendlier: keep digits, compress underscores, and strip trailing underscores. --- .../report_downstream_test_diffs.py | 67 ++++++++++++------- 1 file changed, 43 insertions(+), 24 deletions(-) rename {tests => scripts}/report_downstream_test_diffs.py (53%) diff --git a/tests/report_downstream_test_diffs.py b/scripts/report_downstream_test_diffs.py similarity index 53% rename from tests/report_downstream_test_diffs.py rename to scripts/report_downstream_test_diffs.py index 964bfad3f..8750ae9c1 100644 --- a/tests/report_downstream_test_diffs.py +++ b/scripts/report_downstream_test_diffs.py @@ -3,16 +3,22 @@ import os import re import subprocess -import tempfile +import typing from collections import namedtuple +from playwright.path_utils import get_file_dirname + +_dirname = get_file_dirname() + TestCase = namedtuple("TestCase", ["api", "file", "test"]) -def pytest_test_cases(): +def pytest_test_cases() -> typing.Generator[TestCase, None, None]: p = subprocess.run( ["pytest", "--browser", "chromium", "--collect-only", "-q"], + cwd=_dirname / ".." / "tests", stdout=subprocess.PIPE, + check=True, ) regex = ( r"tests/(?Pa?sync)/test_(?P.*)\.py::test_(?P.*)\[chromium\]" @@ -24,32 +30,44 @@ def pytest_test_cases(): ) -def jest_test_cases(playwright_js_path): - env = os.environ.copy() - env["REPORT_ONLY"] = "true" - with tempfile.NamedTemporaryFile() as results: - subprocess.run( - ["npx", "jest", "--json", "--outputFile", results.name], - env=env, - cwd=playwright_js_path, - stderr=subprocess.DEVNULL, - ) +def jest_test_cases(playwright_js_path: str) -> typing.Generator[TestCase, None, None]: + p = subprocess.run( + [ + "node", + os.path.join("test", "runner"), + "test", + "--trial-run", + "--reporter", + "json", + ], + cwd=playwright_js_path, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=True, + ) + + tests = json.loads(p.stdout.decode()) + for test in [*tests["pending"], *tests["passes"], *tests["failures"]]: + regex = r"(.*/)?(?P[^/]+)\.spec\.[jt]s$" + + match = re.match(regex, test["file"]) + if not match: + continue + + file = match.group("file") - for test_suite in json.load(results)["testResults"]: - regex = r"(.*/)?(?P[^/]+)\.spec\.[jt]s$" - file = re.match(regex, test_suite["name"]).group("file") - for assertion in test_suite["assertionResults"]: - yield TestCase("sync", normalized(file), normalized(assertion["title"])) - yield TestCase( - "async", normalized(file), normalized(assertion["title"]) - ) + yield TestCase("sync", normalized(file), normalized(test["title"])) + yield TestCase("async", normalized(file), normalized(test["title"])) -def normalized(original): - return re.sub(r"[^a-z_]", "_", original, flags=re.IGNORECASE) +def normalized(original: str) -> str: + cleaned = re.sub(r"[^a-z0-9_]", "_", original, flags=re.IGNORECASE) + cleaned = re.sub(r"[_]+", "_", cleaned) + cleaned = cleaned.strip("_") + return cleaned -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "--playwright-js-path", @@ -69,12 +87,13 @@ def main(): javascript_tests = set(jest_test_cases(args.playwright_js_path)) if args.api: - javascript_tests = set([x for x in javascript_tests if x[0] == args.api]) + javascript_tests = set([x for x in javascript_tests if x.api == args.api]) missing = javascript_tests.difference(python_tests) found = javascript_tests.intersection(python_tests) print("MISSING, MISPELLED, OR MISNAMED:") + print("=" * 80) for (api, file, test) in sorted(missing): print(f"{api}/test_{file}.py::test_{test}")