Resolving conflicts

Author: gavin-aguiar

.ci/perf_tests/dockerfiles/perf_tests.Dockerfile

@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION=3.8
 
-FROM mcr.microsoft.com/azure-functions/python:3.0.15418-python$PYTHON_VERSION
+FROM mcr.microsoft.com/azure-functions/python:4-python$PYTHON_VERSION
 
 # Mounting local machines azure-functions-python-worker and azure-functions-python-library onto it
 RUN rm -rf /azure-functions-host/workers/python/${PYTHON_VERSION}/LINUX/X64/azure_functions_worker

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ dev, *dev, main*, release* ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ dev ]
+  schedule:
+    - cron: '25 11 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/perf-testing-setup.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test_to_run: [ SyncHttpTriggerHelloWorld, SyncHttpTriggerWithSyncRequests, AsyncHttpTriggerWithAsyncRequest, SyncHttpTriggerCPUIntensive ] #, SyncPutBlobAsBytesReturnHttpResponse, SyncGetBlobAsBytesReturnHttpResponse ]
+        test_to_run: [ SyncHttpTriggerHelloWorld ]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ env.PYTHON_VERSION }}
@@ -27,6 +27,14 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: '3.1.405'
+    - name: Set up Dotnet 5.0.x
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '5.0.x'
+    - name: Set up Dotnet 6.x
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '6.x'
     - name: Setup k6 for throughput testing
       run: |
         cd $GITHUB_WORKSPACE

CODEOWNERS

@@ -10,4 +10,4 @@
 # For all file changes, github would automatically
 # include the following people in the PRs.
 
-* @vrdmr @gavin-aguiar @YunchuWang
+* @vrdmr @gavin-aguiar @YunchuWang @pdthummar

azure_functions_worker/constants.py

@@ -44,7 +44,7 @@
 PYTHON_THREADPOOL_THREAD_COUNT_MAX_37 = 32
 
 PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = False
-PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = True
+PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True
 

azure_functions_worker/loader.py

@@ -46,7 +46,9 @@ def uninstall() -> None:
 
 @attach_message_to_exception(
     expt_type=ImportError,
-    message=f'Troubleshooting Guide: {MODULE_NOT_FOUND_TS_URL}'
+    message=f'Please check the requirements.txt file for the missing module. '
+            f'For more info, please refer the troubleshooting'
+            f' guide: {MODULE_NOT_FOUND_TS_URL} '
 )
 def load_function(name: str, directory: str, script_file: str,
                   entry_point: typing.Optional[str]):

azure_functions_worker/testutils_lc.py

@@ -1,28 +1,34 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-
-from typing import Dict
-
 import base64
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
+import tempfile
 import time
 import uuid
+from io import BytesIO
+from typing import Dict
+from urllib.request import urlopen
+from zipfile import ZipFile
 
+import requests
 from Crypto.Cipher import AES
 from Crypto.Hash.SHA256 import SHA256Hash
 from Crypto.Util.Padding import pad
-import requests
 
 # Linux Consumption Testing Constants
 _DOCKER_PATH = "DOCKER_PATH"
 _DOCKER_DEFAULT_PATH = "docker"
 _MESH_IMAGE_URL = "https://mcr.microsoft.com/v2/azure-functions/mesh/tags/list"
 _MESH_IMAGE_REPO = "mcr.microsoft.com/azure-functions/mesh"
 _DUMMY_CONT_KEY = "MDEyMzQ1Njc4OUFCQ0RFRjAxMjM0NTY3ODlBQkNERUY="
+_FUNC_GITHUB_ZIP = "https://github.com/Azure/azure-functions-python-library" \
+                   "/archive/refs/heads/dev.zip"
+_FUNC_FILE_NAME = "azure-functions-python-library-dev"
 
 
 class LinuxConsumptionWebHostController:
@@ -80,9 +86,9 @@ def assign_container(self, env: Dict[str, str] = {}):
                                f' stdout: {stdout}')
 
     def send_request(
-        self,
-        req: requests.Request,
-        ses: requests.Session = None
+            self,
+            req: requests.Request,
+            ses: requests.Session = None
     ) -> requests.Response:
         """Send a request with authorization token. Return a Response object"""
         session = ses
@@ -129,6 +135,12 @@ def _find_latest_mesh_image(cls,
         cls._mesh_images[host_major] = image_tag
         return image_tag
 
+    @staticmethod
+    def _download_azure_functions() -> str:
+        with urlopen(_FUNC_GITHUB_ZIP) as zipresp:
+            with ZipFile(BytesIO(zipresp.read())) as zfile:
+                zfile.extractall(tempfile.gettempdir())
+
     def spawn_container(self,
                         image: str,
                         env: Dict[str, str] = {}) -> int:
@@ -137,10 +149,18 @@ def spawn_container(self,
         """
         # Construct environment variables and start the docker container
         worker_path = os.path.dirname(__file__)
+        library_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME,
+                                    'azure', 'functions')
+        self._download_azure_functions()
+
         container_worker_path = (
             f"/azure-functions-host/workers/python/{self._py_version}/"
             "LINUX/X64/azure_functions_worker"
         )
+        container_library_path = (
+            f"/azure-functions-host/workers/python/{self._py_version}/"
+            "LINUX/X64/azure/functions"
+        )
 
         run_cmd = []
         run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"])
@@ -150,7 +170,9 @@ def spawn_container(self,
         run_cmd.extend(["-e", f"CONTAINER_NAME={self._uuid}"])
         run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"])
         run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"])
+        run_cmd.extend(["-e", "PYTHON_ISOLATE_WORKER_DEPENDENCIES=1"])
         run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}'])
+        run_cmd.extend(["-v", f'{library_path}:{container_library_path}'])
 
         for key, value in env.items():
             run_cmd.extend(["-e", f"{key}={value}"])
@@ -181,7 +203,7 @@ def spawn_container(self,
         self._ports[self._uuid] = port_number
 
         # Wait for three seconds for the container to be in ready state
-        time.sleep(3)
+        time.sleep(6)
         return port_number
 
     def get_container_logs(self) -> str:
@@ -259,6 +281,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         logs = self.get_container_logs()
         self.safe_kill_container()
+        shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME))
 
         if traceback:
             print(f'Test failed with container logs: {logs}',

tests/endtoend/test_linux_consumption.py

@@ -1,15 +1,16 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-from unittest import TestCase, skipIf
-
 import os
 import sys
-from requests import Request
+from unittest import TestCase, skipIf
 
 from azure_functions_worker.testutils_lc import (
     LinuxConsumptionWebHostController
 )
 from azure_functions_worker.utils.common import is_python_version
+from requests import Request
+
+_DEFAULT_HOST_VERSION = "3"
 
 
 @skipIf(is_python_version('3.10'),
@@ -42,7 +43,8 @@ def test_placeholder_mode_root_returns_ok(self):
         """In any circumstances, a placeholder container should returns 200
         even when it is not specialized.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             req = Request('GET', ctrl.url)
             resp = ctrl.send_request(req)
             self.assertTrue(resp.ok)
@@ -51,7 +53,8 @@ def test_http_no_auth(self):
         """An HttpTrigger function app with 'azure-functions' library
         should return 200.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("HttpNoAuth")
@@ -60,6 +63,8 @@ def test_http_no_auth(self):
             resp = ctrl.send_request(req)
             self.assertEqual(resp.status_code, 200)
 
+    @skipIf(is_python_version('3.7'),
+            "Skip the tests for Python 3.7.")
     def test_common_libraries(self):
         """A function app with the following requirements.txt:
 
@@ -73,7 +78,8 @@ def test_common_libraries(self):
 
         should return 200 after importing all libraries.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("CommonLibraries")
@@ -98,7 +104,8 @@ def test_new_protobuf(self):
 
         should return 200 after importing all libraries.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf")
@@ -123,10 +130,11 @@ def test_old_protobuf(self):
 
         should return 200 after importing all libraries.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
-                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf")
+                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf")
             })
             req = Request('GET', f'{ctrl.url}/api/HttpTrigger')
             resp = ctrl.send_request(req)
@@ -144,7 +152,8 @@ def test_debug_logging_disabled(self):
         should return 200 and by default customer debug logging should be
         disabled.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("EnableDebugLogging")
@@ -170,7 +179,8 @@ def test_debug_logging_enabled(self):
         should return 200 and with customer debug logging enabled, debug logs
         should be written to container logs.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url(