Merge branch 'dev' into patch-1

vrdmr · web-flow · commit 22644565f7db · 2022-06-30T10:51:49.000-05:00
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ What are the supported Python versions?
 |----------------------------------|------------|------------|------------|------------|-------------|
 | Azure Functions 2.0 (deprecated) | ✔          | ✔          | -          | -          | -           |
 | Azure Functions 3.0              | ✔          | ✔          | ✔          | ✔          | -           |
-| Azure Functions 4.0              | -          | ✔          | ✔          | ✔          | preview     |
+| Azure Functions 4.0              | -          | ✔          | ✔          | ✔          | coming soon     |
 
 For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page.
 
diff --git a/SECURITY.md b/SECURITY.md
@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->
diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py
@@ -43,8 +43,7 @@
 PYTHON_THREADPOOL_THREAD_COUNT_MAX = sys.maxsize
 PYTHON_THREADPOOL_THREAD_COUNT_MAX_37 = 32
 
-PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = False
-PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False
+PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = True
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True
 
diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py
@@ -29,7 +29,7 @@
 from .logging import disable_console_logging, enable_console_logging
 from .logging import enable_debug_logging_recommendation
 from .logging import (logger, error_logger, is_system_log_category,
-                      CONSOLE_LOG_PREFIX)
+                      CONSOLE_LOG_PREFIX, format_exception)
 from .utils.common import get_app_setting, is_envvar_true
 from .utils.dependency import DependencyManager
 from .utils.tracing import marshall_exception_trace
@@ -745,7 +745,9 @@ def gen(resp_queue):
             if ex is grpc_req_stream:
                 # Yes, this is how grpc_req_stream iterator exits.
                 return
-            error_logger.exception('unhandled error in gRPC thread')
+            error_logger.exception(
+                'unhandled error in gRPC thread. Exception: {0}'.format(
+                    format_exception(ex)))
             raise
 
 
diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py
@@ -5,8 +5,6 @@
 import pathlib
 import typing
 
-from azure.functions import DataType, Function
-
 from . import bindings as bindings_utils
 from . import protos
 from ._thirdparty import typing_inspect
@@ -216,8 +214,7 @@ def validate_function_params(params: dict, bound_params: dict,
                         param_bind_type, param_py_type)
 
                 if not checks_out:
-                    if binding.data_type is not DataType(
-                            protos.BindingInfo.undefined):
+                    if binding.data_type is not protos.BindingInfo.undefined:
                         raise FunctionLoadError(
                             func_name,
                             f'{param.name!r} binding type "{binding.type}" '
@@ -357,7 +354,7 @@ def add_function(self, function_id: str,
                                                       output_types, return_type)
 
     def add_indexed_function(self, function_id: str,
-                             function: Function):
+                             function):
         func = function.get_user_function()
         func_name = function.get_function_name()
         return_binding_name: typing.Optional[str] = None
diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py
@@ -10,9 +10,7 @@
 import sys
 import uuid
 from os import PathLike, fspath
-from typing import List, Optional, Dict
-
-from azure.functions import Function, FunctionApp
+from typing import Optional, Dict
 
 from . import protos, functions
 from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \
@@ -47,7 +45,7 @@ def uninstall() -> None:
     pass
 
 
-def build_binding_protos(indexed_function: Function) -> Dict:
+def build_binding_protos(indexed_function) -> Dict:
     binding_protos = {}
     for binding in indexed_function.get_bindings():
         binding_protos[binding.name] = protos.BindingInfo(
@@ -59,7 +57,7 @@ def build_binding_protos(indexed_function: Function) -> Dict:
 
 
 def process_indexed_function(functions_registry: functions.Registry,
-                             indexed_functions: List[Function]):
+                             indexed_functions):
     fx_metadata_results = []
     for indexed_function in indexed_functions:
         function_id = str(uuid.uuid4())
@@ -141,10 +139,11 @@ def load_function(name: str, directory: str, script_file: str,
     expt_type=ImportError,
     message=f'Troubleshooting Guide: {MODULE_NOT_FOUND_TS_URL}'
 )
-def index_function_app(function_path: str) -> List[Function]:
+def index_function_app(function_path: str):
     module_name = pathlib.Path(function_path).stem
     imported_module = importlib.import_module(module_name)
 
+    from azure.functions import FunctionApp
     app: Optional[FunctionApp] = None
     for i in imported_module.__dir__():
         if isinstance(getattr(imported_module, i, None), FunctionApp):
diff --git a/azure_functions_worker/logging.py b/azure_functions_worker/logging.py
@@ -4,6 +4,7 @@
 import logging
 import logging.handlers
 import sys
+import traceback
 from typing import Optional
 
 # Logging Prefixes
@@ -20,6 +21,13 @@
 error_handler: Optional[logging.Handler] = None
 
 
+def format_exception(exception):
+    msg = str(exception) + "\n"
+    msg += ''.join(traceback.format_exception(
+        etype=type(exception), value=exception, tb=exception.__traceback__))
+    return msg
+
+
 def setup(log_level, log_destination):
     # Since handler and error_handler are moved to the global scope,
     # before assigning to these handlers, we should define 'global' keyword
diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py
@@ -36,7 +36,7 @@ def main():
 
     from . import logging
     from ._thirdparty import aio_compat
-    from .logging import error_logger, logger
+    from .logging import error_logger, logger, format_exception
 
     args = parse_args()
     logging.setup(log_level=args.log_level, log_destination=args.log_to)
@@ -48,8 +48,10 @@ def main():
     try:
         return aio_compat.run(start_async(
             args.host, args.port, args.worker_id, args.request_id))
-    except Exception:
-        error_logger.exception('unhandled error in functions worker')
+    except Exception as ex:
+        error_logger.exception(
+            'unhandled error in functions worker: {0}'.format(
+                format_exception(ex)))
         raise
 
 
diff --git a/azure_functions_worker/testutils_lc.py b/azure_functions_worker/testutils_lc.py
@@ -139,16 +139,6 @@ def _find_latest_mesh_image(cls,
         cls._mesh_images[host_major] = image_tag
         return image_tag
 
-    def _delete_init_file_from_azure_dir(self):
-        init_file_path = f"/azure-functions-host/workers/python/" \
-                         f"{self._py_version}/LINUX/X64/azure/__init__.py"
-        run_rm_cmd = [self._docker_cmd, "exec", self._uuid]
-        run_rm_cmd.extend(["rm", init_file_path])
-
-        subprocess.run(args=run_rm_cmd,
-                       stdout=subprocess.PIPE,
-                       stderr=subprocess.PIPE)
-
     @staticmethod
     def _download_azure_functions() -> str:
         with urlopen(_FUNC_GITHUB_ZIP) as zipresp:
@@ -201,9 +191,6 @@ def spawn_container(self,
                                f' {image} with uuid {self._uuid}.'
                                f' stderr: {run_process.stderr}')
 
-        # Remove once worker version 4.0.0 is released
-        self._delete_init_file_from_azure_dir()
-
         # Wait for three seconds for the port to expose
         time.sleep(3)
 
diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py
@@ -1,21 +1,21 @@
-from azure_functions_worker.utils.common import is_true_like
-from typing import List, Optional
-from types import ModuleType
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
 import importlib
 import inspect
 import os
 import re
 import sys
+from types import ModuleType
+from typing import List, Optional
 
-from ..logging import logger
+from azure_functions_worker.utils.common import is_true_like
 from ..constants import (
     AZURE_WEBJOBS_SCRIPT_ROOT,
     CONTAINER_NAME,
     PYTHON_ISOLATE_WORKER_DEPENDENCIES,
-    PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT,
-    PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310
+    PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
 )
-from ..utils.common import is_python_version
+from ..logging import logger
 from ..utils.wrappers import enable_feature_by
 
 
@@ -75,12 +75,7 @@ def is_in_linux_consumption(cls):
     @classmethod
     @enable_feature_by(
         flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES,
-        flag_default=(
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if
-            is_python_version('3.10') else
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
-        )
-    )
+        flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT)
     def use_worker_dependencies(cls):
         """Switch the sys.path and ensure the worker imports are loaded from
         Worker's dependenciess.
@@ -106,12 +101,7 @@ def use_worker_dependencies(cls):
     @classmethod
     @enable_feature_by(
         flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES,
-        flag_default=(
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if
-            is_python_version('3.10') else
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
-        )
-    )
+        flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT)
     def prioritize_customer_dependencies(cls, cx_working_dir=None):
         """Switch the sys.path and ensure the customer's code import are loaded
         from CX's deppendencies.
@@ -180,11 +170,7 @@ def reload_customer_libraries(cls, cx_working_dir: str):
         """
         use_new_env = os.getenv(PYTHON_ISOLATE_WORKER_DEPENDENCIES)
         if use_new_env is None:
-            use_new = (
-                PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if
-                is_python_version('3.10') else
-                PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
-            )
+            use_new = PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
         else:
             use_new = is_true_like(use_new_env)
 
diff --git a/tests/endtoend/dependency_isolation_functions/dependency_isolation_stein/function_app.py b/tests/endtoend/dependency_isolation_functions/dependency_isolation_stein/function_app.py
diff --git a/tests/endtoend/test_linux_consumption.py b/tests/endtoend/test_linux_consumption.py
@@ -11,7 +11,7 @@
 )
 from azure_functions_worker.utils.common import is_python_version
 
-_DEFAULT_HOST_VERSION = "3"
+_DEFAULT_HOST_VERSION = "4"
 
 
 @skipIf(is_python_version('3.10'),
@@ -64,8 +64,6 @@ def test_http_no_auth(self):
             resp = ctrl.send_request(req)
             self.assertEqual(resp.status_code, 200)
 
-    @skipIf(is_python_version('3.7'),
-            "Skip the tests for Python 3.7.")
     def test_common_libraries(self):
         """A function app with the following requirements.txt:
 
@@ -87,14 +85,14 @@ def test_common_libraries(self):
             })
             req = Request('GET', f'{ctrl.url}/api/HttpTrigger')
             resp = ctrl.send_request(req)
+            self.assertEqual(resp.status_code, 200)
             content = resp.json()
             self.assertIn('azure.functions', content)
             self.assertIn('azure.storage.blob', content)
             self.assertIn('numpy', content)
             self.assertIn('cryptography', content)
             self.assertIn('pyodbc', content)
             self.assertIn('requests', content)
-            self.assertEqual(resp.status_code, 200)
 
     def test_new_protobuf(self):
         """A function app with the following requirements.txt:
@@ -205,6 +203,25 @@ def test_debug_logging_enabled(self):
             self.assertIn('logging error', func_log)
             self.assertIn('logging debug', func_log)
 
+    def test_pinning_functions_to_older_version(self):
+        """An HttpTrigger function app with 'azure-functions==1.11.1' library
+        should return 200 with the azure functions version set to 1.11.1
+        since dependency isolation is enabled by default for all py versions
+        """
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
+
+            ctrl.assign_container(env={
+                "AzureWebJobsStorage": self._storage,
+                "SCM_RUN_FROM_PACKAGE": self._get_blob_url(
+                    "PinningFunctions"),
+            })
+            req = Request('GET', f'{ctrl.url}/api/HttpTrigger1')
+            resp = ctrl.send_request(req)
+
+            self.assertEqual(resp.status_code, 200)
+            self.assertIn("Func Version: 1.11.1", resp.text)
+
     def _get_blob_url(self, scenario_name: str) -> str:
         return (
             f'https://pythonworker{self._py_shortform}sa.blob.core.windows.net/'
diff --git a/tests/unittests/broken_functions/invalid_datatype/function.json b/tests/unittests/broken_functions/invalid_datatype/function.json
@@ -0,0 +1,11 @@
+{
+  "scriptFile": "main.py",
+  "bindings": [
+    {
+      "type": "httpTrigger",
+      "direction": "in",
+      "dataType" : "string",
+      "name": "req"
+    }
+  ]
+}
diff --git a/tests/unittests/broken_functions/invalid_datatype/main.py b/tests/unittests/broken_functions/invalid_datatype/main.py
@@ -0,0 +1,7 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import azure.functions as azf
+
+
+def main(req: azf.HttpResponse):
+    return 'This function should fail!!'
diff --git a/tests/unittests/test_broken_functions.py b/tests/unittests/test_broken_functions.py
diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py
diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py
diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py
diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py
