From a02d2ddf283dbacfdd4d4bcd8a163dd03a4e0c29 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 22 Nov 2022 14:19:57 -0600 Subject: [PATCH 01/30] Moved consumption tests in separate pipeline --- .github/workflows/ci_consumption_workflow.yml | 46 +++++++++++++++++++ .github/workflows/ci_e2e_workflow.yml | 12 ----- ...{ut_ci_workflow.yml => ci_ut_workflow.yml} | 0 .../test_linux_consumption.py | 0 4 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ci_consumption_workflow.yml rename .github/workflows/{ut_ci_workflow.yml => ci_ut_workflow.yml} (100%) rename tests/{endtoend => consumption_tests}/test_linux_consumption.py (100%) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml new file mode 100644 index 000000000..cc10788ab --- /dev/null +++ b/.github/workflows/ci_consumption_workflow.yml @@ -0,0 +1,46 @@ +# This workflow will run all tests in tests/consumption_tests in Docker using a consumption image +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CI Consumption tests + +on: + workflow_dispatch: + push: + branches: [ dev, master, main, release/* ] + pull_request: + branches: [ dev, master, main, release/* ] + +jobs: + build: + name: "Python Consumption CI Run" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ 3.7, 3.8, 3.9, "3.10" ] + + steps: + - name: Running 3.7 Tests + if: matrix.python-version == 3.7 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} + run: | + python -m pytest -n auto --instafail tests/consumption_tests + - name: Running 3.8 Tests + if: matrix.python-version == 3.8 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} + run: | + python -m pytest -n auto --instafail tests/consumption_tests + - name: Running 3.9 Tests + if: matrix.python-version == 3.9 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} + run: | + python -m pytest -n auto --instafail tests/consumption_tests + - name: Running 3.10 Tests + if: matrix.python-version == 3.10 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} + run: | + python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index 7347d53bb..096a7ef74 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -65,18 +65,6 @@ jobs: retry 5 python setup.py build retry 5 python setup.py webhost --branch-name=dev retry 5 python setup.py extension - - name: Running 3.6 Tests - if: matrix.python-version == 3.6 - env: - AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString36 }} - AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString36 }} - AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString36 }} - AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString36 }} - AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString36 }} - AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString36 }} - AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString36 }} - run: | - python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ci_ut_workflow.yml similarity index 100% rename from .github/workflows/ut_ci_workflow.yml rename to .github/workflows/ci_ut_workflow.yml diff --git a/tests/endtoend/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py similarity index 100% rename from tests/endtoend/test_linux_consumption.py rename to tests/consumption_tests/test_linux_consumption.py From 4961ed54ea2a7a7d41c214fe07390c3f15d9ab5d Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 22 Nov 2022 14:27:59 -0600 Subject: [PATCH 02/30] Installing pytest --- .github/workflows/ci_consumption_workflow.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index cc10788ab..92bc7d350 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -20,27 +20,29 @@ jobs: python-version: [ 3.7, 3.8, 3.9, "3.10" ] steps: + - name: Install dependencies + run: python -m pip install pytest pytest-xdist pytest-instafail - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests + python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.8 Tests if: matrix.python-version == 3.8 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests + python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.9 Tests if: matrix.python-version == 3.9 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests + python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.10 Tests if: matrix.python-version == 3.10 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file + python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file From e8c8ba2df767e3f5349d5948135fcbe095ad5970 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 22 Nov 2022 17:24:28 -0600 Subject: [PATCH 03/30] Updated dependencies --- .github/workflows/ci_consumption_workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 92bc7d350..faaf31e71 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -21,28 +21,28 @@ jobs: steps: - name: Install dependencies - run: python -m pip install pytest pytest-xdist pytest-instafail + run: python -m pip install pytest - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests + python -m pytest tests/consumption_tests - name: Running 3.8 Tests if: matrix.python-version == 3.8 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests + python -m pytest tests/consumption_tests - name: Running 3.9 Tests if: matrix.python-version == 3.9 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests + python -m pytest tests/consumption_tests - name: Running 3.10 Tests if: matrix.python-version == 3.10 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} run: | - python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file + python -m pytest tests/consumption_tests \ No newline at end of file From 9b4d16a418e0e4458e0ae044afbbaffac6208ad4 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 01:23:02 -0600 Subject: [PATCH 04/30] Added xdist to pytest --- .github/workflows/ci_consumption_workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index faaf31e71..ba580a548 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -27,22 +27,22 @@ jobs: env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} run: | - python -m pytest tests/consumption_tests + python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.8 Tests if: matrix.python-version == 3.8 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} run: | - python -m pytest tests/consumption_tests + python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.9 Tests if: matrix.python-version == 3.9 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} run: | - python -m pytest tests/consumption_tests + python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.10 Tests if: matrix.python-version == 3.10 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} run: | - python -m pytest tests/consumption_tests \ No newline at end of file + python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file From 83f502a8736ed0ab1ae51a37e9c5476fbbda1cb0 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 01:53:33 -0600 Subject: [PATCH 05/30] Installing dependencies --- .github/workflows/ci_consumption_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index ba580a548..15c64712c 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Install dependencies - run: python -m pip install pytest + run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From a8bfc3b9d528b25c8e1383e0830477253fa8e418 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 05:57:22 -0600 Subject: [PATCH 06/30] Installing additional dependencies --- .github/workflows/ci_consumption_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 15c64712c..c0c216a11 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Install dependencies - run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] + run: python -m pip install pytest pytest-xdist pytest-instafail - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From a3e137b9ae94b8ef5d6c3f5429be89df15344cfb Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 06:29:12 -0600 Subject: [PATCH 07/30] Formatted yml file --- .github/workflows/ci_consumption_workflow.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index c0c216a11..0793a104c 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -26,23 +26,19 @@ jobs: if: matrix.python-version == 3.7 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} - run: | - python -m pytest -n auto --instafail tests/consumption_tests + run: python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.8 Tests if: matrix.python-version == 3.8 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} - run: | - python -m pytest -n auto --instafail tests/consumption_tests + run: python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.9 Tests if: matrix.python-version == 3.9 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} - run: | - python -m pytest -n auto --instafail tests/consumption_tests + run: python -m pytest -n auto --instafail tests/consumption_tests - name: Running 3.10 Tests if: matrix.python-version == 3.10 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} - run: | - python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file + run: python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file From 55544d01aec13685f9d63bf4ec744ec8f7ebb587 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 06:41:08 -0600 Subject: [PATCH 08/30] Added __init__ --- tests/consumption_tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/consumption_tests/__init__.py diff --git a/tests/consumption_tests/__init__.py b/tests/consumption_tests/__init__.py new file mode 100644 index 000000000..e69de29bb From db572cf19e71c6288b39604d5cb024cc47e11071 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 30 Nov 2022 11:50:34 -0600 Subject: [PATCH 09/30] list dirs --- .github/workflows/ci_consumption_workflow.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 0793a104c..deeae3708 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Install dependencies run: python -m pip install pytest pytest-xdist pytest-instafail + - name: List dirs + run: ls -lR - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: @@ -41,4 +43,4 @@ jobs: if: matrix.python-version == 3.10 env: AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} - run: python -m pytest -n auto --instafail tests/consumption_tests \ No newline at end of file + run: python -m pytest -n auto --instafail tests/consumption_tests From c9ef0ea37b9e2b0736e09770eeb44cb2610e7e77 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 30 Nov 2022 11:55:49 -0600 Subject: [PATCH 10/30] Update ci_consumption_workflow.yml --- .github/workflows/ci_consumption_workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index deeae3708..87b226dbf 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Install dependencies run: python -m pip install pytest pytest-xdist pytest-instafail + - name: Where am I + run: pwd - name: List dirs run: ls -lR - name: Running 3.7 Tests From 9bd608a1917399d0d7a0f975476e731c741edcb5 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 07:02:27 -0600 Subject: [PATCH 11/30] Updated yml to checkout code --- .github/workflows/ci_consumption_workflow.yml | 6 ++++++ tests/consumption_tests/__init__.py | 0 2 files changed, 6 insertions(+) delete mode 100644 tests/consumption_tests/__init__.py diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 87b226dbf..7a40d4440 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -20,6 +20,12 @@ jobs: python-version: [ 3.7, 3.8, 3.9, "3.10" ] steps: + - name: Checkout code. + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - name: Install dependencies run: python -m pip install pytest pytest-xdist pytest-instafail - name: Where am I diff --git a/tests/consumption_tests/__init__.py b/tests/consumption_tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 From ab30ccac9c85548c597a648bf72cc3e515881119 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 07:27:04 -0600 Subject: [PATCH 12/30] Added requests to dependencies --- .github/workflows/ci_consumption_workflow.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 7a40d4440..7f2183009 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -27,11 +27,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: python -m pip install pytest pytest-xdist pytest-instafail - - name: Where am I - run: pwd - - name: List dirs - run: ls -lR + run: python -m pip install pytest pytest-xdist pytest-instafail requests - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From 4719b036ea443f76e7fd970784e4e593da13b76b Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 07:39:41 -0600 Subject: [PATCH 13/30] Added crypto to dependencies --- .github/workflows/ci_consumption_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 7f2183009..1e841f4ba 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -27,7 +27,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: python -m pip install pytest pytest-xdist pytest-instafail requests + run: python -m pip install pytest pytest-xdist pytest-instafail requests crypto - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From 74a0292f608f76a6ab193d12a1affe44e0f12dda Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 07:49:59 -0600 Subject: [PATCH 14/30] Install all dependencies --- .github/workflows/ci_consumption_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 1e841f4ba..39014ace9 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -27,7 +27,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: python -m pip install pytest pytest-xdist pytest-instafail requests crypto + run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From 50499a887beeb0bc90ac545ebe8e4ddbbd80afd4 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 08:14:10 -0600 Subject: [PATCH 15/30] Unskipped tests for py310 --- tests/consumption_tests/test_linux_consumption.py | 4 +--- tests/utils/testutils_lc.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 83c4b94ca..ef11ca5d8 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import os import sys -from unittest import TestCase, skipIf +from unittest import TestCase from requests import Request @@ -13,8 +13,6 @@ _DEFAULT_HOST_VERSION = "4" -@skipIf(sys.version_info >= (3, 10, 0), - "Skip the tests for Python 3.10 and above") class TestLinuxConsumption(TestCase): """Test worker behaviors on specific scenarios. diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index e24996b2d..99be6f7de 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -151,7 +151,7 @@ def _find_latest_mesh_image(cls, @staticmethod def _download_azure_functions() -> str: with urlopen(_FUNC_GITHUB_ZIP) as zipresp: - with ZipFile(BytesIO(zipresp.read())) as zfile: + with ZipFile(BytesIO(zipresp.read()), 'w') as zfile: zfile.extractall(tempfile.gettempdir()) def spawn_container(self, From 52ab60bbd3b5b013cff5d8f4c47f2644d44f0f3f Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 08:29:56 -0600 Subject: [PATCH 16/30] Undo last change --- tests/utils/testutils_lc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 99be6f7de..0a4bd8c41 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -151,7 +151,7 @@ def _find_latest_mesh_image(cls, @staticmethod def _download_azure_functions() -> str: with urlopen(_FUNC_GITHUB_ZIP) as zipresp: - with ZipFile(BytesIO(zipresp.read()), 'w') as zfile: + with ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall(tempfile.gettempdir()) def spawn_container(self, @@ -295,7 +295,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), True) if traceback: print(f'Test failed with container logs: {logs}', From 1124ea69e7128086740108b72c36eb60377a7e8e Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 09:18:31 -0600 Subject: [PATCH 17/30] Give permissions to temp dir --- tests/utils/testutils_lc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 0a4bd8c41..5856e49e3 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -152,6 +152,7 @@ def _find_latest_mesh_image(cls, def _download_azure_functions() -> str: with urlopen(_FUNC_GITHUB_ZIP) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: + os.chmod(tempfile.gettempdir(), 700) zfile.extractall(tempfile.gettempdir()) def spawn_container(self, @@ -295,7 +296,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), True) + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) if traceback: print(f'Test failed with container logs: {logs}', From f554c18c8279f8dc7bab58a8487d1716a6a27ce2 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 09:51:30 -0600 Subject: [PATCH 18/30] Temp: check user --- .github/workflows/ci_consumption_workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 39014ace9..bc4f85003 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -28,6 +28,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] + - name: who am i + run: whoami + - name: temp dir permissions + run: ls -la /tmp - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From e26cab4707934b81a8a52ff895e3b3d47d0b55da Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 13:56:30 -0600 Subject: [PATCH 19/30] Testing build worker --- .github/workflows/ci_consumption_workflow.yml | 8 +++----- tests/utils/testutils_lc.py | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index bc4f85003..dc32fc41e 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -27,11 +27,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] - - name: who am i - run: whoami - - name: temp dir permissions - run: ls -la /tmp + run: | + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] + python setup.py build - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 5856e49e3..e24996b2d 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -152,7 +152,6 @@ def _find_latest_mesh_image(cls, def _download_azure_functions() -> str: with urlopen(_FUNC_GITHUB_ZIP) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: - os.chmod(tempfile.gettempdir(), 700) zfile.extractall(tempfile.gettempdir()) def spawn_container(self, From 920d2c9482e144b803cc0e5f578f9020a0f1aca3 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 14:35:15 -0600 Subject: [PATCH 20/30] Fix syntax error --- .github/workflows/ci_consumption_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index dc32fc41e..0d2bd4487 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -28,8 +28,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] - python setup.py build + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] + python setup.py build - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From e4ec078f3f89969c716c0577d8944e30974a05f7 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 14:57:21 -0600 Subject: [PATCH 21/30] Ignore errors in cleanup --- tests/consumption_tests/test_linux_consumption.py | 1 + tests/utils/testutils_lc.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index ef11ca5d8..1bc4e9e99 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -211,6 +211,7 @@ def test_pinning_functions_to_older_version(self): "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url( "PinningFunctions"), + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger1') resp = ctrl.send_request(req) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index e24996b2d..1835005b9 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -183,7 +183,6 @@ def spawn_container(self, run_cmd.extend(["-e", f"CONTAINER_NAME={self._uuid}"]) run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"]) run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"]) - run_cmd.extend(["-e", "PYTHON_ISOLATE_WORKER_DEPENDENCIES=1"]) run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}']) run_cmd.extend(["-v", f'{library_path}:{container_library_path}']) @@ -295,7 +294,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), + ignore_errors=True) if traceback: print(f'Test failed with container logs: {logs}', From 7f3773d729b7a66e121b2e2cb1c5d5f69323077f Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 15:52:51 -0600 Subject: [PATCH 22/30] Added dependency isolation flag --- tests/consumption_tests/test_linux_consumption.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 1bc4e9e99..7035314ff 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -103,7 +103,8 @@ def test_new_protobuf(self): self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, - "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf") + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf"), + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger') resp = ctrl.send_request(req) @@ -130,7 +131,8 @@ def test_old_protobuf(self): self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, - "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf") + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf"), + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger') resp = ctrl.send_request(req) From 68dce802f063b6af62205004b33bd8db8e689144 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 17:55:14 -0600 Subject: [PATCH 23/30] Building webhost --- .github/workflows/ci_consumption_workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml index 0d2bd4487..aaa30ced4 100644 --- a/.github/workflows/ci_consumption_workflow.yml +++ b/.github/workflows/ci_consumption_workflow.yml @@ -30,6 +30,8 @@ jobs: run: | python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] python setup.py build + python setup.py webhost --branch-name=dev + python setup.py extension - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: From 52dc96ad32b14196a9e4c009b30b5a4ca44fb56c Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 18:22:10 -0600 Subject: [PATCH 24/30] Deleting temp dir --- tests/utils/testutils_lc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 1835005b9..d8416921f 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -150,10 +150,15 @@ def _find_latest_mesh_image(cls, @staticmethod def _download_azure_functions() -> str: + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), + ignore_errors=True) with urlopen(_FUNC_GITHUB_ZIP) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall(tempfile.gettempdir()) + if not os.path.exists(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)): + raise FileNotFoundError + def spawn_container(self, image: str, env: Dict[str, str] = {}) -> int: From 1ad2fc669f9a864fff582640be985aaadc43ba81 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 29 Nov 2022 18:58:38 -0600 Subject: [PATCH 25/30] Adding logs --- tests/utils/testutils_lc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index d8416921f..0ce225dc9 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -157,7 +157,9 @@ def _download_azure_functions() -> str: zfile.extractall(tempfile.gettempdir()) if not os.path.exists(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)): - raise FileNotFoundError + raise RuntimeError( + f'{_FUNC_FILE_NAME} not found in {tempfile.gettempdir()}' + f'List: {os.listdir(tempfile.gettempdir())}') def spawn_container(self, image: str, From bacdc7feb50ad21e418a617e6ae60955ca37cb9b Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Wed, 30 Nov 2022 07:25:57 -0600 Subject: [PATCH 26/30] Check if dir is empty --- tests/utils/testutils_lc.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 0ce225dc9..800111e63 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -156,11 +156,15 @@ def _download_azure_functions() -> str: with ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall(tempfile.gettempdir()) - if not os.path.exists(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)): + + lib_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME) + if not os.path.exists(lib_path): raise RuntimeError( - f'{_FUNC_FILE_NAME} not found in {tempfile.gettempdir()}' - f'List: {os.listdir(tempfile.gettempdir())}') + f'{_FUNC_FILE_NAME} not found in {tempfile.gettempdir()}') + if not os.listdir(lib_path): + raise RuntimeError(f'{_FUNC_FILE_NAME} empty') + def spawn_container(self, image: str, env: Dict[str, str] = {}) -> int: From 28268e0bdf306ededb2d199e176861c7a87c7a8c Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Wed, 30 Nov 2022 07:39:23 -0600 Subject: [PATCH 27/30] Removing library mounting --- tests/utils/testutils_lc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 800111e63..3ae6aa234 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -181,10 +181,10 @@ def spawn_container(self, f"/azure-functions-host/workers/python/{self._py_version}/" "LINUX/X64/azure_functions_worker" ) - container_library_path = ( - f"/azure-functions-host/workers/python/{self._py_version}/" - "LINUX/X64/azure/functions" - ) + # container_library_path = ( + # f"/azure-functions-host/workers/python/{self._py_version}/" + # "LINUX/X64/azure/functions" + # ) run_cmd = [] run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"]) @@ -195,7 +195,7 @@ def spawn_container(self, run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"]) run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"]) run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}']) - run_cmd.extend(["-v", f'{library_path}:{container_library_path}']) + # run_cmd.extend(["-v", f'{library_path}:{container_library_path}']) for key, value in env.items(): run_cmd.extend(["-e", f"{key}={value}"]) From 8e965d09f109928f9c1ee22ef3cc1fce511eba60 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Wed, 30 Nov 2022 08:02:57 -0600 Subject: [PATCH 28/30] Fixing flake8 test --- tests/utils/testutils_lc.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 3ae6aa234..7bde36c0f 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -150,21 +150,10 @@ def _find_latest_mesh_image(cls, @staticmethod def _download_azure_functions() -> str: - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), - ignore_errors=True) with urlopen(_FUNC_GITHUB_ZIP) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall(tempfile.gettempdir()) - - lib_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME) - if not os.path.exists(lib_path): - raise RuntimeError( - f'{_FUNC_FILE_NAME} not found in {tempfile.gettempdir()}') - - if not os.listdir(lib_path): - raise RuntimeError(f'{_FUNC_FILE_NAME} empty') - def spawn_container(self, image: str, env: Dict[str, str] = {}) -> int: @@ -173,18 +162,14 @@ def spawn_container(self, """ # Construct environment variables and start the docker container worker_path = os.path.join(PROJECT_ROOT, 'azure_functions_worker') - library_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME, - 'azure', 'functions') + + # TODO: Mount library in docker container self._download_azure_functions() container_worker_path = ( f"/azure-functions-host/workers/python/{self._py_version}/" "LINUX/X64/azure_functions_worker" ) - # container_library_path = ( - # f"/azure-functions-host/workers/python/{self._py_version}/" - # "LINUX/X64/azure/functions" - # ) run_cmd = [] run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"]) @@ -195,7 +180,6 @@ def spawn_container(self, run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"]) run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"]) run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}']) - # run_cmd.extend(["-v", f'{library_path}:{container_library_path}']) for key, value in env.items(): run_cmd.extend(["-e", f"{key}={value}"]) @@ -305,8 +289,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), - ignore_errors=True) + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) if traceback: print(f'Test failed with container logs: {logs}', From 27e3332b66f7a644065350e466e0b7f87eee412a Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Wed, 30 Nov 2022 08:42:02 -0600 Subject: [PATCH 29/30] Skipped downloading library --- tests/utils/testutils_lc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index 7bde36c0f..68598e7dd 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -164,7 +164,7 @@ def spawn_container(self, worker_path = os.path.join(PROJECT_ROOT, 'azure_functions_worker') # TODO: Mount library in docker container - self._download_azure_functions() + # self._download_azure_functions() container_worker_path = ( f"/azure-functions-host/workers/python/{self._py_version}/" @@ -289,7 +289,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), + ignore_errors=True) if traceback: print(f'Test failed with container logs: {logs}', From 5f5af69df29b2eb57c974f433dec72f2c21a08de Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 6 Dec 2022 16:16:40 -0600 Subject: [PATCH 30/30] Skipping protobuf tests for py310 --- tests/consumption_tests/test_linux_consumption.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 7035314ff..681ae866f 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import os import sys -from unittest import TestCase +from unittest import TestCase, skipIf from requests import Request @@ -90,6 +90,8 @@ def test_common_libraries(self): self.assertIn('pyodbc', content) self.assertIn('requests', content) + @skipIf(sys.version_info.minor == 10, + "Protobuf pinning fails during remote build") def test_new_protobuf(self): """A function app with the following requirements.txt: @@ -118,6 +120,8 @@ def test_new_protobuf(self): self.assertEqual(content['google.protobuf'], '3.15.8') self.assertEqual(content['grpc'], '1.33.2') + @skipIf(sys.version_info.minor == 10, + "Protobuf pinning fails during remote build") def test_old_protobuf(self): """A function app with the following requirements.txt: