From 703907bcdf0b60074a8894ead98ae24dc516fc14 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 18 Apr 2019 08:58:40 -0700 Subject: [PATCH 01/11] bpo-33608: Normalize atomic macros so that they all expect an atomic struct --- Include/internal/pycore_atomic.h | 60 +++++++++++++++++++------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h index b3ec44c1bcfee1..336bc3fec27e5d 100644 --- a/Include/internal/pycore_atomic.h +++ b/Include/internal/pycore_atomic.h @@ -261,13 +261,13 @@ typedef struct _Py_atomic_int { #define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ switch (ORDER) { \ case _Py_memory_order_acquire: \ - _InterlockedExchange64_HLEAcquire((__int64 volatile*)ATOMIC_VAL, (__int64)NEW_VAL); \ + _InterlockedExchange64_HLEAcquire((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ break; \ case _Py_memory_order_release: \ - _InterlockedExchange64_HLERelease((__int64 volatile*)ATOMIC_VAL, (__int64)NEW_VAL); \ + _InterlockedExchange64_HLERelease((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ break; \ default: \ - _InterlockedExchange64((__int64 volatile*)ATOMIC_VAL, (__int64)NEW_VAL); \ + _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ break; \ } #else @@ -277,13 +277,13 @@ typedef struct _Py_atomic_int { #define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ switch (ORDER) { \ case _Py_memory_order_acquire: \ - _InterlockedExchange_HLEAcquire((volatile long*)ATOMIC_VAL, (int)NEW_VAL); \ + _InterlockedExchange_HLEAcquire((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ break; \ case _Py_memory_order_release: \ - _InterlockedExchange_HLERelease((volatile long*)ATOMIC_VAL, (int)NEW_VAL); \ + _InterlockedExchange_HLERelease((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ break; \ default: \ - _InterlockedExchange((volatile long*)ATOMIC_VAL, (int)NEW_VAL); \ + _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ break; \ } @@ -292,7 +292,7 @@ typedef struct _Py_atomic_int { gil_created() uses -1 as a sentinel value, if this returns a uintptr_t it will do an unsigned compare and crash */ -inline intptr_t _Py_atomic_load_64bit(volatile uintptr_t* value, int order) { +inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) { __int64 old; switch (order) { case _Py_memory_order_acquire: @@ -323,11 +323,14 @@ inline intptr_t _Py_atomic_load_64bit(volatile uintptr_t* value, int order) { return old; } +#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ + _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER)) + #else -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) *(ATOMIC_VAL) +#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) #endif -inline int _Py_atomic_load_32bit(volatile int* value, int order) { +inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { long old; switch (order) { case _Py_memory_order_acquire: @@ -358,16 +361,19 @@ inline int _Py_atomic_load_32bit(volatile int* value, int order) { return old; } +#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ + _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER)) + #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ if (sizeof((ATOMIC_VAL)->_value) == 8) { \ - _Py_atomic_store_64bit((volatile long long*)&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) } else { \ - _Py_atomic_store_32bit((volatile long*)&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) } + _Py_atomic_store_64bit((ATOMIC_VAL), NEW_VAL, ORDER) } else { \ + _Py_atomic_store_32bit((ATOMIC_VAL), NEW_VAL, ORDER) } #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ ( \ sizeof((ATOMIC_VAL)->_value) == 8 ? \ - _Py_atomic_load_64bit((volatile long long*)&((ATOMIC_VAL)->_value), ORDER) : \ - _Py_atomic_load_32bit((volatile long*)&((ATOMIC_VAL)->_value), ORDER) \ + _Py_atomic_load_64bit((ATOMIC_VAL), ORDER) : \ + _Py_atomic_load_32bit((ATOMIC_VAL), ORDER) \ ) #elif defined(_M_ARM) || defined(_M_ARM64) typedef enum _Py_memory_order { @@ -422,7 +428,7 @@ typedef struct _Py_atomic_int { gil_created() uses -1 as a sentinel value, if this returns a uintptr_t it will do an unsigned compare and crash */ -inline intptr_t _Py_atomic_load_64bit(volatile uintptr_t* value, int order) { +inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) { uintptr_t old; switch (order) { case _Py_memory_order_acquire: @@ -453,11 +459,14 @@ inline intptr_t _Py_atomic_load_64bit(volatile uintptr_t* value, int order) { return old; } +#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ + _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER)) + #else -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) *(ATOMIC_VAL) +#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) #endif -inline int _Py_atomic_load_32bit(volatile int* value, int order) { +inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { int old; switch (order) { case _Py_memory_order_acquire: @@ -488,16 +497,19 @@ inline int _Py_atomic_load_32bit(volatile int* value, int order) { return old; } +#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ + _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER)) + #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ if (sizeof((ATOMIC_VAL)->_value) == 8) { \ - _Py_atomic_store_64bit(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) } else { \ - _Py_atomic_store_32bit(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) } + _Py_atomic_store_64bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } else { \ + _Py_atomic_store_32bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ ( \ sizeof((ATOMIC_VAL)->_value) == 8 ? \ - _Py_atomic_load_64bit(&((ATOMIC_VAL)->_value), ORDER) : \ - _Py_atomic_load_32bit(&((ATOMIC_VAL)->_value), ORDER) \ + _Py_atomic_load_64bit((ATOMIC_VAL), (ORDER)) : \ + _Py_atomic_load_32bit((ATOMIC_VAL), (ORDER)) \ ) #endif #else /* !gcc x86 !_msc_ver */ @@ -529,16 +541,16 @@ typedef struct _Py_atomic_int { /* Standardized shortcuts. */ #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ - _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst) + _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_seq_cst) #define _Py_atomic_load(ATOMIC_VAL) \ - _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst) + _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_seq_cst) /* Python-local extensions */ #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ - _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed) + _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_relaxed) #define _Py_atomic_load_relaxed(ATOMIC_VAL) \ - _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed) + _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_relaxed) #ifdef __cplusplus } From a61436b1c309cace27af6ed15428e53eb6074705 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Thu, 18 Apr 2019 15:06:53 -0700 Subject: [PATCH 02/11] windows arm32 buildbot changes --- Tools/buildbot/test.bat | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 40ffc7efdd2d6c..1c68811b677e4b 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -5,9 +5,11 @@ setlocal set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 +set arm32_ssh= :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-arm32" (set arm32_ssh=true) & shift & goto CheckOpts if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts @@ -16,4 +18,17 @@ if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% +set rt_args=%rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% +if "%arm32_ssh%"=="true" goto :Arm32Ssh + +call "%here%..\..\PCbuild\rt.bat" %rt_args% +exit /b 0 + +:Arm32Ssh +if "%SSH_SERVER%"=="" (echo SSH_SERVER environment variable must be set & exit 127) +if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python) +if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here..\..\) +ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +scp "%PYTHON_SOURCE%" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%" +ssh %SSH_SERVER% call "%REMOTE_PYTHON_DIR%\PCbuild\rt.bat" %rt_args% +exit /b 0 From 520c7bc69b50a61fdee3c94a2d5749e8eabfe90d Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Thu, 18 Apr 2019 16:01:45 -0700 Subject: [PATCH 03/11] save progress --- Tools/buildbot/test.bat | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 1c68811b677e4b..e6bfe6c70a861d 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -6,11 +6,12 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 set arm32_ssh= +set dparam= :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-arm32" (set arm32_ssh=true) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-d" (set rt_opts=%rt_opts% %1) & (set dparam=-d)& shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts @@ -25,10 +26,18 @@ call "%here%..\..\PCbuild\rt.bat" %rt_args% exit /b 0 :Arm32Ssh -if "%SSH_SERVER%"=="" (echo SSH_SERVER environment variable must be set & exit 127) -if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python) -if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here..\..\) -ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" -scp "%PYTHON_SOURCE%" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%" +if "%SSH_SERVER%"=="" (echo SSH_SERVER environment variable must be set & exit /b 127) +if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) +if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python\) +if "%REMOTE_PYTHON_SHARE%"=="" (set REMOTE_PYTHON_SHARE=P:\python\) +#ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +#scp "%PYTHON_SOURCE%python.bat" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%python.bat" +#scp -r "%PYTHON_SOURCE%PCBuild" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +#scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" +if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_SHARE% /s/q) +python PC/layout -vv %dparam% -s %PYTHON_SOURCE% -b "%PYTHON_SOURCE%\PCBuild\arm32" -t "%PYTHON_SHARE%\temp" --copy "%PYTHON_SHARE%" --preset-iot --include-tests --include-venv +if NOT EXIST %REMOTE_PYTHON_DIR%\PCbuild (md %REMOTE_PYTHON_DIR%\PCbuild) +copy %PYTHON_SOURCE%\PCBuild\*.bat %REMOTE_PYTHON_DIR%\PCbuild +copy %PYTHON_SOURCE%\PCBuild\*.py %REMOTE_PYTHON_DIR%\PCbuild ssh %SSH_SERVER% call "%REMOTE_PYTHON_DIR%\PCbuild\rt.bat" %rt_args% exit /b 0 From fbf4957702c14e64282c9645986ccc05516d329f Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Mon, 22 Apr 2019 16:05:31 -0700 Subject: [PATCH 04/11] snapshot pc/layout approach --- Tools/buildbot/test.bat | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index e6bfe6c70a861d..1630c30f0224ee 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -6,7 +6,7 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 set arm32_ssh= -set dparam= +set dparam=-d :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts @@ -14,7 +14,7 @@ if "%1"=="-arm32" (set arm32_ssh=true) & shift & goto CheckOpts if "%1"=="-d" (set rt_opts=%rt_opts% %1) & (set dparam=-d)& shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & (dparam=) & shift & goto CheckOpts if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts @@ -29,15 +29,15 @@ exit /b 0 if "%SSH_SERVER%"=="" (echo SSH_SERVER environment variable must be set & exit /b 127) if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python\) -if "%REMOTE_PYTHON_SHARE%"=="" (set REMOTE_PYTHON_SHARE=P:\python\) -#ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" -#scp "%PYTHON_SOURCE%python.bat" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%python.bat" -#scp -r "%PYTHON_SOURCE%PCBuild" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" -#scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" -if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_SHARE% /s/q) -python PC/layout -vv %dparam% -s %PYTHON_SOURCE% -b "%PYTHON_SOURCE%\PCBuild\arm32" -t "%PYTHON_SHARE%\temp" --copy "%PYTHON_SHARE%" --preset-iot --include-tests --include-venv -if NOT EXIST %REMOTE_PYTHON_DIR%\PCbuild (md %REMOTE_PYTHON_DIR%\PCbuild) -copy %PYTHON_SOURCE%\PCBuild\*.bat %REMOTE_PYTHON_DIR%\PCbuild -copy %PYTHON_SOURCE%\PCBuild\*.py %REMOTE_PYTHON_DIR%\PCbuild -ssh %SSH_SERVER% call "%REMOTE_PYTHON_DIR%\PCbuild\rt.bat" %rt_args% +if "%REMOTE_PYTHON_SHARE%"=="" (set REMOTE_PYTHON_SHARE=P:\python) +REM #ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +REM #scp "%PYTHON_SOURCE%python.bat" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%python.bat" +REM #scp -r "%PYTHON_SOURCE%PCBuild" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +REM #scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" +if EXIST %REMOTE_PYTHON_SHARE% (rd %REMOTE_PYTHON_SHARE% /s/q) +python.exe PC/layout -vv %dparam% -s %PYTHON_SOURCE% -b "%PYTHON_SOURCE%\PCBuild\arm32" -t "%REMOTE_PYTHON_SHARE%\temp" --copy "%REMOTE_PYTHON_SHARE%" --preset-iot --include-tests --include-venv +if NOT EXIST %REMOTE_PYTHON_SHARE%\PCbuild (md %REMOTE_PYTHON_SHARE%\PCbuild) +copy %PYTHON_SOURCE%PCBuild\*.bat %REMOTE_PYTHON_SHARE%\PCbuild +copy %PYTHON_SOURCE%PCBuild\*.py %REMOTE_PYTHON_SHARE%\PCbuild +ssh %SSH_SERVER% call "%REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% exit /b 0 From 2fc1d4a3e516ec27cf168cb423c4c666abc1a814 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Mon, 22 Apr 2019 17:34:55 -0700 Subject: [PATCH 05/11] use ssh and scp --- PCbuild/rt.bat | 1 + Tools/buildbot/test.bat | 34 +++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 212befc95b0698..e603de6d5174f8 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -39,6 +39,7 @@ if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%win32 diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 1630c30f0224ee..3b1bba0544a84a 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -10,7 +10,7 @@ set dparam=-d :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm32" (set arm32_ssh=true) & shift & goto CheckOpts +if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts if "%1"=="-d" (set rt_opts=%rt_opts% %1) & (set dparam=-d)& shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts @@ -26,18 +26,26 @@ call "%here%..\..\PCbuild\rt.bat" %rt_args% exit /b 0 :Arm32Ssh -if "%SSH_SERVER%"=="" (echo SSH_SERVER environment variable must be set & exit /b 127) +if "%SSH_SERVER%"=="" goto :Arm32SshHelp if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python\) -if "%REMOTE_PYTHON_SHARE%"=="" (set REMOTE_PYTHON_SHARE=P:\python) -REM #ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" -REM #scp "%PYTHON_SOURCE%python.bat" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%python.bat" -REM #scp -r "%PYTHON_SOURCE%PCBuild" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" -REM #scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" -if EXIST %REMOTE_PYTHON_SHARE% (rd %REMOTE_PYTHON_SHARE% /s/q) -python.exe PC/layout -vv %dparam% -s %PYTHON_SOURCE% -b "%PYTHON_SOURCE%\PCBuild\arm32" -t "%REMOTE_PYTHON_SHARE%\temp" --copy "%REMOTE_PYTHON_SHARE%" --preset-iot --include-tests --include-venv -if NOT EXIST %REMOTE_PYTHON_SHARE%\PCbuild (md %REMOTE_PYTHON_SHARE%\PCbuild) -copy %PYTHON_SOURCE%PCBuild\*.bat %REMOTE_PYTHON_SHARE%\PCbuild -copy %PYTHON_SOURCE%PCBuild\*.py %REMOTE_PYTHON_SHARE%\PCbuild -ssh %SSH_SERVER% call "%REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% +ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +for /f "USEBACKQ" %%i in (`dir PCbuild\*.py /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.exe /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.pyd /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.dll /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +scp -r "%PYTHON_SOURCE%Include" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Include" +scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" +ssh %SSH_SERVER% "%REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% exit /b 0 + +:Arm32SshHelp +echo SSH_SERVER environment variable must be set to administrator@[ip address] +echo where [ip address] is the address of a Windows IoT Core ARM32 device. +echo. +echo The test worker should have the SSH agent running. +echo Also a key must be created with ssh-keygen and added to both the buildbot worker machine +echo and the ARM32 worker device: see https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh +exit /b 127 \ No newline at end of file From 16d94a3e20744856256025e19cbef1767b539446 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Thu, 18 Apr 2019 15:06:53 -0700 Subject: [PATCH 06/11] windows arm32 buildbot changes --- PCbuild/rt.bat | 1 + Tools/buildbot/test.bat | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 212befc95b0698..e603de6d5174f8 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -39,6 +39,7 @@ if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%win32 diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 40ffc7efdd2d6c..3b1bba0544a84a 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -5,15 +5,47 @@ setlocal set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 +set arm32_ssh= +set dparam=-d :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts +if "%1"=="-d" (set rt_opts=%rt_opts% %1) & (set dparam=-d)& shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & (dparam=) & shift & goto CheckOpts if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% +set rt_args=%rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% +if "%arm32_ssh%"=="true" goto :Arm32Ssh + +call "%here%..\..\PCbuild\rt.bat" %rt_args% +exit /b 0 + +:Arm32Ssh +if "%SSH_SERVER%"=="" goto :Arm32SshHelp +if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) +if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python\) +ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +for /f "USEBACKQ" %%i in (`dir PCbuild\*.py /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.exe /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.pyd /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.dll /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +scp -r "%PYTHON_SOURCE%Include" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Include" +scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" +ssh %SSH_SERVER% "%REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% +exit /b 0 + +:Arm32SshHelp +echo SSH_SERVER environment variable must be set to administrator@[ip address] +echo where [ip address] is the address of a Windows IoT Core ARM32 device. +echo. +echo The test worker should have the SSH agent running. +echo Also a key must be created with ssh-keygen and added to both the buildbot worker machine +echo and the ARM32 worker device: see https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh +exit /b 127 \ No newline at end of file From 8a2245219793eaa8871a74b66d633e872f99227c Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Thu, 18 Apr 2019 15:06:53 -0700 Subject: [PATCH 07/11] windows arm32 buildbot changes --- Tools/buildbot/test.bat | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 40ffc7efdd2d6c..1c68811b677e4b 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -5,9 +5,11 @@ setlocal set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 +set arm32_ssh= :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-arm32" (set arm32_ssh=true) & shift & goto CheckOpts if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts @@ -16,4 +18,17 @@ if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% +set rt_args=%rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% +if "%arm32_ssh%"=="true" goto :Arm32Ssh + +call "%here%..\..\PCbuild\rt.bat" %rt_args% +exit /b 0 + +:Arm32Ssh +if "%SSH_SERVER%"=="" (echo SSH_SERVER environment variable must be set & exit 127) +if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python) +if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here..\..\) +ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +scp "%PYTHON_SOURCE%" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%" +ssh %SSH_SERVER% call "%REMOTE_PYTHON_DIR%\PCbuild\rt.bat" %rt_args% +exit /b 0 From 38a6930ce17ef4cfd10860c4a1711f197b49d619 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Mon, 29 Apr 2019 15:55:25 -0700 Subject: [PATCH 08/11] change -u options --- Tools/buildbot/test.bat | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 3b1bba0544a84a..54aa2fe7cc1bf1 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -19,13 +19,14 @@ if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -set rt_args=%rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% if "%arm32_ssh%"=="true" goto :Arm32Ssh -call "%here%..\..\PCbuild\rt.bat" %rt_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% exit /b 0 :Arm32Ssh +set dashU=-unetwork,decimal,subprocess,urlfetch,tzdata +set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% if "%SSH_SERVER%"=="" goto :Arm32SshHelp if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python\) From c17411955b1d8443139fe7757da88aef5b952343 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Mon, 29 Apr 2019 16:10:44 -0700 Subject: [PATCH 09/11] cleanup dparam --- Tools/buildbot/test.bat | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 56a5efc64649cd..764dcd8583d556 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -6,15 +6,14 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 set arm32_ssh= -set dparam=-d :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & (set dparam=-d)& shift & goto CheckOpts +if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & (dparam=) & shift & goto CheckOpts +if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts From 21960371fb5e3ce9b7049709bcf22218a8624d84 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Fri, 3 May 2019 10:21:48 -0700 Subject: [PATCH 10/11] set TEMP dir to path without symbolic link --- Tools/buildbot/test.bat | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 764dcd8583d556..5a05d417c245ee 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -25,10 +25,10 @@ exit /b 0 :Arm32Ssh set dashU=-unetwork,decimal,subprocess,urlfetch,tzdata -set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% if "%SSH_SERVER%"=="" goto :Arm32SshHelp if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) -if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=c:\python\) +if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) +set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" @@ -38,6 +38,8 @@ for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.pyd /b`) do @scp PCBuild\arm32\%% for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.dll /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" scp -r "%PYTHON_SOURCE%Include" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Include" scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" + +set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% %TEMP_ARGS% ssh %SSH_SERVER% "%REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% exit /b 0 From 3a1a391ea11a0f2d630c365603580932887aba2b Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Fri, 3 May 2019 13:01:03 -0700 Subject: [PATCH 11/11] set TEMP environment variable --- Tools/buildbot/test.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 5a05d417c245ee..7815c55c152bf6 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -31,6 +31,7 @@ if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%temp" for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" for /f "USEBACKQ" %%i in (`dir PCbuild\*.py /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.exe /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" @@ -40,7 +41,7 @@ scp -r "%PYTHON_SOURCE%Include" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Include" scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% %TEMP_ARGS% -ssh %SSH_SERVER% "%REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% +ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp & %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% exit /b 0 :Arm32SshHelp