From 6bd1482704dab92891b3e3b7ffe7380b36ab4146 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 13 Sep 2023 01:02:28 +0200 Subject: [PATCH] gh-109276: libregrtest: WASM use stdout for JSON On Emscripten and WASI, or if --python command line is used, libregrtest now writes JSON into stdout, instead of using a name file. * Add JsonFileType.STDOUT. * Remove JsonFileType.FILENAME. * test.pythoninfo logs environment variables related to cross-compilation and running Python on Emscripten/WASI. --- Lib/test/libregrtest/run_workers.py | 26 ++++++++++++-------------- Lib/test/libregrtest/runtests.py | 27 ++++++++++++++++----------- Lib/test/libregrtest/worker.py | 14 +++++++------- Lib/test/pythoninfo.py | 6 +++++- 4 files changed, 40 insertions(+), 33 deletions(-) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index a6bebccd0c7585..f99ca343eefb06 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -21,7 +21,7 @@ from .runtests import RunTests, JsonFile, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( - StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX, + StrPath, StrJSON, TestName, MS_WINDOWS, format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP @@ -225,16 +225,9 @@ def create_stdout(self, stack: contextlib.ExitStack) -> TextIO: def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextIO | None]: """Create JSON file.""" - json_file_use_filename = self.runtests.json_file_use_filename() - if json_file_use_filename: - # create an empty file to make the creation atomic - # (to prevent races with other worker threads) - prefix = TMP_PREFIX + 'json_' - json_fd, json_filename = tempfile.mkstemp(prefix=prefix) - os.close(json_fd) - - stack.callback(os_helper.unlink, json_filename) - json_file = JsonFile(json_filename, JsonFileType.FILENAME) + json_file_use_stdout = self.runtests.json_file_use_stdout() + if json_file_use_stdout: + json_file = JsonFile(None, JsonFileType.STDOUT) json_tmpfile = None else: json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') @@ -300,11 +293,14 @@ def read_stdout(self, stdout_file: TextIO) -> str: f"Cannot read process stdout: {exc}", None) def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, - stdout: str) -> TestResult: + stdout: str) -> tuple[TestResult, str]: try: if json_tmpfile is not None: json_tmpfile.seek(0) worker_json: StrJSON = json_tmpfile.read() + elif json_file.file_type == JsonFileType.STDOUT: + stdout, _, worker_json = stdout.rpartition("\n") + stdout = stdout.rstrip() else: with json_file.open(encoding='utf8') as json_fp: worker_json: StrJSON = json_fp.read() @@ -319,7 +315,7 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, raise WorkerError(self.test_name, "empty JSON", stdout) try: - return TestResult.from_json(worker_json) + result = TestResult.from_json(worker_json) except Exception as exc: # gh-101634: Catch UnicodeDecodeError if stdout cannot be # decoded from encoding @@ -327,6 +323,8 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, raise WorkerError(self.test_name, err_msg, stdout, state=State.MULTIPROCESSING_ERROR) + return (result, stdout) + def _runtest(self, test_name: TestName) -> MultiprocessResult: with contextlib.ExitStack() as stack: stdout_file = self.create_stdout(stack) @@ -341,7 +339,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: if retcode is None: raise WorkerError(self.test_name, None, stdout, state=State.TIMEOUT) - result = self.read_json(json_file, json_tmpfile, stdout) + result, stdout = self.read_json(json_file, json_tmpfile, stdout) if retcode != 0: raise WorkerError(self.test_name, f"Exit code {retcode}", stdout) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 62a0a7e20c7b8b..aee0ab6fd6e38f 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -14,13 +14,16 @@ class JsonFileType: UNIX_FD = "UNIX_FD" WINDOWS_HANDLE = "WINDOWS_HANDLE" - FILENAME = "FILENAME" + STDOUT = "STDOUT" @dataclasses.dataclass(slots=True, frozen=True) class JsonFile: - # See RunTests.json_file_use_filename() - file: int | StrPath + # file type depends on file_type: + # - UNIX_FD: file descriptor (int) + # - WINDOWS_HANDLE: handle (int) + # - STDOUT: use process stdout (None) + file: int | None file_type: str def configure_subprocess(self, popen_kwargs: dict) -> None: @@ -33,9 +36,6 @@ def configure_subprocess(self, popen_kwargs: dict) -> None: startupinfo = subprocess.STARTUPINFO() startupinfo.lpAttributeList = {"handle_list": [self.file]} popen_kwargs['startupinfo'] = startupinfo - case JsonFileType.FILENAME: - # Filename: nothing to do to - pass @contextlib.contextmanager def inherit_subprocess(self): @@ -49,6 +49,9 @@ def inherit_subprocess(self): yield def open(self, mode='r', *, encoding): + if self.file_type == JsonFileType.STDOUT: + raise ValueError("for STDOUT file type, just use sys.stdout") + file = self.file if self.file_type == JsonFileType.WINDOWS_HANDLE: import msvcrt @@ -123,11 +126,13 @@ def as_json(self) -> StrJSON: def from_json(worker_json: StrJSON) -> 'RunTests': return json.loads(worker_json, object_hook=_decode_runtests) - def json_file_use_filename(self) -> bool: - # json_file type depends on the platform: - # - Unix: file descriptor (int) - # - Windows: handle (int) - # - Emscripten/WASI or if --python is used: filename (str) + def json_file_use_stdout(self) -> bool: + # Use STDOUT in two cases: + # + # - If --python command line option is used; + # - On Emscripten and WASI. + # + # On other platforms, UNIX_FD or WINDOWS_HANDLE can be used. return ( bool(self.python_cmd) or support.is_emscripten diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index ae3dcb2b06ed36..168803c5d9451f 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,7 +7,7 @@ from test.support import os_helper from .setup import setup_process, setup_test_dir -from .runtests import RunTests, JsonFile +from .runtests import RunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( StrPath, StrJSON, FilterTuple, @@ -67,10 +67,6 @@ def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests - # json_file type depends on the platform: - # - Unix: file descriptor (int) - # - Windows: handle (int) - # - Emscripten/WASI or if --python is used: filename (str) json_file: JsonFile = runtests.json_file setup_test_dir(runtests.test_dir) @@ -85,8 +81,12 @@ def worker_process(worker_json: StrJSON) -> NoReturn: result = run_single_test(test_name, runtests) - with json_file.open('w', encoding='utf-8') as json_fp: - result.write_json_into(json_fp) + if json_file.file_type == JsonFileType.STDOUT: + print() + result.write_json_into(sys.stdout) + else: + with json_file.open('w', encoding='utf-8') as json_fp: + result.write_json_into(json_fp) sys.exit(0) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index ea3a78a341c43f..f16b7986995c38 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -268,6 +268,7 @@ def format_groups(groups): "ARCHFLAGS", "ARFLAGS", "AUDIODEV", + "BUILDPYTHON", "CC", "CFLAGS", "COLUMNS", @@ -320,6 +321,7 @@ def format_groups(groups): "VIRTUAL_ENV", "WAYLAND_DISPLAY", "WINDIR", + "_PYTHON_HOSTRUNNER", "_PYTHON_HOST_PLATFORM", "_PYTHON_PROJECT_BASE", "_PYTHON_SYSCONFIGDATA_NAME", @@ -335,7 +337,8 @@ def format_groups(groups): for name, value in os.environ.items(): uname = name.upper() if (uname in ENV_VARS - # Copy PYTHON* and LC_* variables + # Copy PYTHON* variables like PYTHONPATH + # Copy LC_* variables like LC_ALL or uname.startswith(("PYTHON", "LC_")) # Visual Studio: VS140COMNTOOLS or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): @@ -500,6 +503,7 @@ def collect_sysconfig(info_add): 'CFLAGS', 'CFLAGSFORSHARED', 'CONFIG_ARGS', + 'HOSTRUNNER', 'HOST_GNU_TYPE', 'MACHDEP', 'MULTIARCH',