diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml new file mode 100644 index 000000000..aaa30ced4 --- /dev/null +++ b/.github/workflows/ci_consumption_workflow.yml @@ -0,0 +1,54 @@ +# This workflow will run all tests in tests/consumption_tests in Docker using a consumption image +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CI Consumption tests + +on: + workflow_dispatch: + push: + branches: [ dev, master, main, release/* ] + pull_request: + branches: [ dev, master, main, release/* ] + +jobs: + build: + name: "Python Consumption CI Run" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ 3.7, 3.8, 3.9, "3.10" ] + + steps: + - name: Checkout code. + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] + python setup.py build + python setup.py webhost --branch-name=dev + python setup.py extension + - name: Running 3.7 Tests + if: matrix.python-version == 3.7 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} + run: python -m pytest -n auto --instafail tests/consumption_tests + - name: Running 3.8 Tests + if: matrix.python-version == 3.8 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} + run: python -m pytest -n auto --instafail tests/consumption_tests + - name: Running 3.9 Tests + if: matrix.python-version == 3.9 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} + run: python -m pytest -n auto --instafail tests/consumption_tests + - name: Running 3.10 Tests + if: matrix.python-version == 3.10 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} + run: python -m pytest -n auto --instafail tests/consumption_tests diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index 7347d53bb..096a7ef74 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -65,18 +65,6 @@ jobs: retry 5 python setup.py build retry 5 python setup.py webhost --branch-name=dev retry 5 python setup.py extension - - name: Running 3.6 Tests - if: matrix.python-version == 3.6 - env: - AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString36 }} - AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString36 }} - AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString36 }} - AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString36 }} - AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString36 }} - AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString36 }} - AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString36 }} - run: | - python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend - name: Running 3.7 Tests if: matrix.python-version == 3.7 env: diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ci_ut_workflow.yml similarity index 100% rename from .github/workflows/ut_ci_workflow.yml rename to .github/workflows/ci_ut_workflow.yml diff --git a/tests/endtoend/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py similarity index 95% rename from tests/endtoend/test_linux_consumption.py rename to tests/consumption_tests/test_linux_consumption.py index 83c4b94ca..681ae866f 100644 --- a/tests/endtoend/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -13,8 +13,6 @@ _DEFAULT_HOST_VERSION = "4" -@skipIf(sys.version_info >= (3, 10, 0), - "Skip the tests for Python 3.10 and above") class TestLinuxConsumption(TestCase): """Test worker behaviors on specific scenarios. @@ -92,6 +90,8 @@ def test_common_libraries(self): self.assertIn('pyodbc', content) self.assertIn('requests', content) + @skipIf(sys.version_info.minor == 10, + "Protobuf pinning fails during remote build") def test_new_protobuf(self): """A function app with the following requirements.txt: @@ -105,7 +105,8 @@ def test_new_protobuf(self): self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, - "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf") + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf"), + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger') resp = ctrl.send_request(req) @@ -119,6 +120,8 @@ def test_new_protobuf(self): self.assertEqual(content['google.protobuf'], '3.15.8') self.assertEqual(content['grpc'], '1.33.2') + @skipIf(sys.version_info.minor == 10, + "Protobuf pinning fails during remote build") def test_old_protobuf(self): """A function app with the following requirements.txt: @@ -132,7 +135,8 @@ def test_old_protobuf(self): self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, - "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf") + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf"), + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger') resp = ctrl.send_request(req) @@ -213,6 +217,7 @@ def test_pinning_functions_to_older_version(self): "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url( "PinningFunctions"), + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger1') resp = ctrl.send_request(req) diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index e24996b2d..68598e7dd 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -162,18 +162,14 @@ def spawn_container(self, """ # Construct environment variables and start the docker container worker_path = os.path.join(PROJECT_ROOT, 'azure_functions_worker') - library_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME, - 'azure', 'functions') - self._download_azure_functions() + + # TODO: Mount library in docker container + # self._download_azure_functions() container_worker_path = ( f"/azure-functions-host/workers/python/{self._py_version}/" "LINUX/X64/azure_functions_worker" ) - container_library_path = ( - f"/azure-functions-host/workers/python/{self._py_version}/" - "LINUX/X64/azure/functions" - ) run_cmd = [] run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"]) @@ -183,9 +179,7 @@ def spawn_container(self, run_cmd.extend(["-e", f"CONTAINER_NAME={self._uuid}"]) run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"]) run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"]) - run_cmd.extend(["-e", "PYTHON_ISOLATE_WORKER_DEPENDENCIES=1"]) run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}']) - run_cmd.extend(["-v", f'{library_path}:{container_library_path}']) for key, value in env.items(): run_cmd.extend(["-e", f"{key}={value}"]) @@ -295,7 +289,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() - shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME), + ignore_errors=True) if traceback: print(f'Test failed with container logs: {logs}',