From 917055b1e1ab72b9a6aa1890298737ad58ecad94 Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Wed, 19 Jun 2019 14:54:43 -0700 Subject: [PATCH 1/3] bpo-36511 add arm32 buildbot diagnostic messages --- Tools/buildbot/remoteDeploy.bat | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Tools/buildbot/remoteDeploy.bat b/Tools/buildbot/remoteDeploy.bat index ac1e38b9617c94..4368a067b34f6b 100644 --- a/Tools/buildbot/remoteDeploy.bat +++ b/Tools/buildbot/remoteDeploy.bat @@ -13,13 +13,23 @@ if "%arm32_ssh%"=="true" goto :Arm32Ssh :Arm32Ssh if "%SSH_SERVER%"=="" goto :Arm32SshHelp + +rem Make sure we have SSH and SCP if "%SSH%"=="" if EXIST %WINDIR%\System32\OpenSSH\ssh.exe (set SSH=%WINDIR%\System32\OpenSSH\ssh.exe) if "%SCP%"=="" if EXIST %WINDIR%\System32\OpenSSH\scp.exe (set SCP=%WINDIR%\System32\OpenSSH\scp.exe) echo SSH = %SSH% echo SCP = %SCP% +if not EXIST "%SSH%" (echo ERROR: SSH not found) & exit /b 1 +if not EXIST "%SCP%" (echo ERROR: SCP not found) & exit /b 1 +%SSH% %SSH_SERVER% echo Testing SSH... +if %ERRORLEVEL% NEQ 0 echo SSH does not work + if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DIR%\) +echo PYTHON_SOURCE = %PYTHON_SOURCE% +echo REMOTE_PYTHON_DIR = %REMOTE_PYTHON_DIR% + %SSH% %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" %SSH% %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" %SSH% %SSH_SERVER% "md %REMOTE_PYTHON_DIR%temp" From 7a93c7a1e13b44295e3d59000f9765a9864e3a1f Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Wed, 19 Jun 2019 15:54:02 -0700 Subject: [PATCH 2/3] find ssh even if running in 32-bit environment --- Tools/buildbot/remoteDeploy.bat | 42 +++++++++++++---------------- Tools/buildbot/remotePythonInfo.bat | 8 ++++-- Tools/buildbot/test.bat | 8 ++++-- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/Tools/buildbot/remoteDeploy.bat b/Tools/buildbot/remoteDeploy.bat index 4368a067b34f6b..722fcde1aa0c39 100644 --- a/Tools/buildbot/remoteDeploy.bat +++ b/Tools/buildbot/remoteDeploy.bat @@ -2,6 +2,7 @@ rem Used by the buildbot "remotedeploy" step. setlocal +set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH set here=%~dp0 set arm32_ssh= @@ -14,15 +15,8 @@ if "%arm32_ssh%"=="true" goto :Arm32Ssh :Arm32Ssh if "%SSH_SERVER%"=="" goto :Arm32SshHelp -rem Make sure we have SSH and SCP -if "%SSH%"=="" if EXIST %WINDIR%\System32\OpenSSH\ssh.exe (set SSH=%WINDIR%\System32\OpenSSH\ssh.exe) -if "%SCP%"=="" if EXIST %WINDIR%\System32\OpenSSH\scp.exe (set SCP=%WINDIR%\System32\OpenSSH\scp.exe) -echo SSH = %SSH% -echo SCP = %SCP% -if not EXIST "%SSH%" (echo ERROR: SSH not found) & exit /b 1 -if not EXIST "%SCP%" (echo ERROR: SCP not found) & exit /b 1 -%SSH% %SSH_SERVER% echo Testing SSH... -if %ERRORLEVEL% NEQ 0 echo SSH does not work +ssh %SSH_SERVER% echo Make sure we can find SSH and SSH_SERVER variable is valid +if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b 1 if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) @@ -30,21 +24,21 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI echo PYTHON_SOURCE = %PYTHON_SOURCE% echo REMOTE_PYTHON_DIR = %REMOTE_PYTHON_DIR% -%SSH% %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" -%SSH% %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" -%SSH% %SSH_SERVER% "md %REMOTE_PYTHON_DIR%temp" -%SSH% %SSH_SERVER% "md %REMOTE_PYTHON_DIR%Modules" -%SSH% %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PC" -for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @%SCP% PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" -for /f "USEBACKQ" %%i in (`dir PCbuild\*.py /b`) do @%SCP% PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" -for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.exe /b`) do @%SCP% PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" -for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.pyd /b`) do @%SCP% PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" -for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.dll /b`) do @%SCP% PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" -%SCP% -r "%PYTHON_SOURCE%Include" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Include" -%SCP% -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" -%SCP% -r "%PYTHON_SOURCE%Tools" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Tools" -%SCP% "%PYTHON_SOURCE%Modules\Setup" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Modules" -%SCP% "%PYTHON_SOURCE%PC\pyconfig.h" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PC" +ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%temp" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%Modules" +ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PC" +for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +for /f "USEBACKQ" %%i in (`dir PCbuild\*.py /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.exe /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.pyd /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.dll /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32" +scp -r "%PYTHON_SOURCE%Include" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Include" +scp -r "%PYTHON_SOURCE%Lib" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Lib" +scp -r "%PYTHON_SOURCE%Tools" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Tools" +scp "%PYTHON_SOURCE%Modules\Setup" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%Modules" +scp "%PYTHON_SOURCE%PC\pyconfig.h" "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PC" exit /b %ERRORLEVEL% diff --git a/Tools/buildbot/remotePythonInfo.bat b/Tools/buildbot/remotePythonInfo.bat index 263808cfddfd40..7285529bb78da0 100644 --- a/Tools/buildbot/remotePythonInfo.bat +++ b/Tools/buildbot/remotePythonInfo.bat @@ -2,6 +2,7 @@ rem Used by the buildbot "remotedeploy" step. setlocal +set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH set here=%~dp0 set arm32_ssh= set suffix=_d @@ -18,10 +19,13 @@ if "%arm32_ssh%"=="true" goto :Arm32Ssh :Arm32Ssh if "%SSH_SERVER%"=="" goto :Arm32SshHelp -if "%SSH%"=="" if EXIST %WINDIR%\System32\OpenSSH\ssh.exe (set SSH=%WINDIR%\System32\OpenSSH\ssh.exe) + +ssh %SSH_SERVER% echo Make sure we can find SSH and SSH_SERVER variable is valid +if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b 1 + set PYTHON_EXE=%prefix%\python%suffix%.exe echo on -%SSH% %SSH_SERVER% %PYTHON_EXE% -m test.pythoninfo +ssh %SSH_SERVER% %PYTHON_EXE% -m test.pythoninfo exit /b %ERRORLEVEL% :Arm32SshHelp diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 685f73d41edc4f..acdcce8489092a 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -2,6 +2,7 @@ rem Used by the buildbot "test" step. setlocal +set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 @@ -31,11 +32,14 @@ if "%SSH_SERVER%"=="" goto :Arm32SshHelp if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DIR%\) -if "%SSH%"=="" if EXIST %WINDIR%\System32\OpenSSH\ssh.exe (set SSH=%WINDIR%\System32\OpenSSH\ssh.exe) + +ssh %SSH_SERVER% echo Make sure we can find SSH and SSH_SERVER variable is valid +if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b 1 + set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% %TEMP_ARGS% -%SSH% %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% +ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% exit /b %ERRORLEVEL% :Arm32SshHelp From 184892abc7af769e8ca87048a5c658ffca66490b Mon Sep 17 00:00:00 2001 From: Paul Monson Date: Wed, 19 Jun 2019 16:27:27 -0700 Subject: [PATCH 3/3] remoteDeploy and remotePythonInfo tested on worker --- Tools/buildbot/remoteDeploy.bat | 4 ++-- Tools/buildbot/remotePythonInfo.bat | 5 +---- Tools/buildbot/test.bat | 5 +---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Tools/buildbot/remoteDeploy.bat b/Tools/buildbot/remoteDeploy.bat index 722fcde1aa0c39..8b0ce239e1a403 100644 --- a/Tools/buildbot/remoteDeploy.bat +++ b/Tools/buildbot/remoteDeploy.bat @@ -2,7 +2,7 @@ rem Used by the buildbot "remotedeploy" step. setlocal -set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH +set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH set here=%~dp0 set arm32_ssh= @@ -16,7 +16,7 @@ if "%arm32_ssh%"=="true" goto :Arm32Ssh if "%SSH_SERVER%"=="" goto :Arm32SshHelp ssh %SSH_SERVER% echo Make sure we can find SSH and SSH_SERVER variable is valid -if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b 1 +if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b %ERRORLEVEL% if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) diff --git a/Tools/buildbot/remotePythonInfo.bat b/Tools/buildbot/remotePythonInfo.bat index 7285529bb78da0..b17717cb018ece 100644 --- a/Tools/buildbot/remotePythonInfo.bat +++ b/Tools/buildbot/remotePythonInfo.bat @@ -2,7 +2,7 @@ rem Used by the buildbot "remotedeploy" step. setlocal -set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH +set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH set here=%~dp0 set arm32_ssh= set suffix=_d @@ -20,9 +20,6 @@ if "%arm32_ssh%"=="true" goto :Arm32Ssh :Arm32Ssh if "%SSH_SERVER%"=="" goto :Arm32SshHelp -ssh %SSH_SERVER% echo Make sure we can find SSH and SSH_SERVER variable is valid -if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b 1 - set PYTHON_EXE=%prefix%\python%suffix%.exe echo on ssh %SSH_SERVER% %PYTHON_EXE% -m test.pythoninfo diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index acdcce8489092a..63b37d3ea90203 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -2,7 +2,7 @@ rem Used by the buildbot "test" step. setlocal -set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH +set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 @@ -33,9 +33,6 @@ if "%PYTHON_SOURCE%"=="" (set PYTHON_SOURCE=%here%..\..\) if "%REMOTE_PYTHON_DIR%"=="" (set REMOTE_PYTHON_DIR=C:\python\) if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DIR%\) -ssh %SSH_SERVER% echo Make sure we can find SSH and SSH_SERVER variable is valid -if %ERRORLEVEL% NEQ 0 (echo SSH does not work) & exit /b 1 - set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% %TEMP_ARGS%