Skip to content

Commit 73fc3db

Browse files
authored
Merge branch 'dev' into pthummar/add_features_e2e_tests
2 parents 120d639 + d844de5 commit 73fc3db

36 files changed

+351
-111
lines changed

.github/workflows/ci_consumption_workflow.yml

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# This workflow will run all tests in tests/consumption_tests in Docker using a consumption image
22
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
33

4-
name: CI Consumption tests
4+
name: CI Consumption E2E tests
55

66
on:
77
workflow_dispatch:
@@ -28,10 +28,7 @@ jobs:
2828
python-version: ${{ matrix.python-version }}
2929
- name: Install dependencies
3030
run: |
31-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
32-
python setup.py build
33-
python setup.py webhost --branch-name=dev
34-
python setup.py extension
31+
python -m pip install -U -e .[dev]
3532
- name: Running 3.7 Tests
3633
if: matrix.python-version == 3.7
3734
env:

.github/workflows/ci_docker_con_workflow.yml

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,7 @@ jobs:
2929
python-version: ${{ matrix.python-version }}
3030
- name: Install dependencies
3131
run: |
32-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
33-
python setup.py build
34-
python setup.py webhost --branch-name=dev
35-
python setup.py extension
32+
python -m pip install -U -e .[dev]
3633
- name: Running 3.7 Tests
3734
if: matrix.python-version == 3.7
3835
env:

.github/workflows/ci_docker_ded_workflow.yml

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,7 @@ jobs:
3030
python-version: ${{ matrix.python-version }}
3131
- name: Install dependencies
3232
run: |
33-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
34-
python setup.py build
35-
python setup.py webhost --branch-name=dev
36-
python setup.py extension
33+
python -m pip install -U -e .[dev]
3734
- name: Running 3.7 Tests
3835
if: matrix.python-version == 3.7
3936
env:

.github/workflows/ci_e2e_workflow.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ jobs:
5858
}
5959
6060
python -m pip install --upgrade pip
61-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
61+
python -m pip install -U -e .[dev]
6262
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
6363
6464
# Retry a couple times to avoid certificate issue
@@ -76,7 +76,7 @@ jobs:
7676
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
7777
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
7878
run: |
79-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
79+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
8080
- name: Running 3.8 Tests
8181
if: matrix.python-version == 3.8
8282
env:
@@ -88,7 +88,7 @@ jobs:
8888
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
8989
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
9090
run: |
91-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
91+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
9292
- name: Running 3.9 Tests
9393
if: matrix.python-version == 3.9
9494
env:
@@ -100,7 +100,7 @@ jobs:
100100
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
101101
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
102102
run: |
103-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
103+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
104104
- name: Running 3.10 Tests
105105
if: matrix.python-version == 3.10
106106
env:
@@ -112,7 +112,7 @@ jobs:
112112
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
113113
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
114114
run: |
115-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
115+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
116116
- name: Running 3.11 Tests
117117
if: matrix.python-version == 3.11
118118
env:
@@ -124,7 +124,7 @@ jobs:
124124
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
125125
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
126126
run: |
127-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
127+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
128128
- name: Codecov
129129
uses: codecov/[email protected]
130130
with:

.github/workflows/ci_ut_workflow.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
}
5656
5757
python -m pip install --upgrade pip
58-
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
58+
python -m pip install -U -e .[dev]
5959
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
6060
6161
# Retry a couple times to avoid certificate issue
@@ -66,7 +66,7 @@ jobs:
6666
env:
6767
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} # needed for installing azure-functions-durable while running setup.py
6868
run: |
69-
python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
69+
python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
7070
- name: Codecov
7171
uses: codecov/codecov-action@v3
7272
with:

azure_functions_worker/bindings/context.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
3+
import threading
4+
from typing import Type
5+
36
from . import TraceContext
47
from . import RetryContext
58

@@ -10,18 +13,24 @@ def __init__(self,
1013
func_name: str,
1114
func_dir: str,
1215
invocation_id: str,
16+
thread_local_storage: Type[threading.local],
1317
trace_context: TraceContext,
1418
retry_context: RetryContext) -> None:
1519
self.__func_name = func_name
1620
self.__func_dir = func_dir
1721
self.__invocation_id = invocation_id
22+
self.__thread_local_storage = thread_local_storage
1823
self.__trace_context = trace_context
1924
self.__retry_context = retry_context
2025

2126
@property
2227
def invocation_id(self) -> str:
2328
return self.__invocation_id
2429

30+
@property
31+
def thread_local_storage(self) -> Type[threading.local]:
32+
return self.__thread_local_storage
33+
2534
@property
2635
def function_name(self) -> str:
2736
return self.__func_name

azure_functions_worker/constants.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@
3030
"""
3131
UNIX_SHARED_MEMORY_DIRECTORIES = "FUNCTIONS_UNIX_SHARED_MEMORY_DIRECTORIES"
3232

33+
# Flag to enable loading functions at init request
34+
PYTHON_LOAD_FUNCTIONS_INIT = "PYTHON_LOAD_FUNCTIONS_INIT"
35+
3336
# Setting Defaults
3437
PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT = 1
3538
PYTHON_THREADPOOL_THREAD_COUNT_MIN = 1

azure_functions_worker/dispatcher.py

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
PYTHON_THREADPOOL_THREAD_COUNT_MAX_37,
2727
PYTHON_THREADPOOL_THREAD_COUNT_MIN,
2828
PYTHON_ENABLE_DEBUG_LOGGING, SCRIPT_FILE_NAME,
29-
PYTHON_LANGUAGE_RUNTIME)
29+
PYTHON_LANGUAGE_RUNTIME, PYTHON_LOAD_FUNCTIONS_INIT)
3030
from .extension import ExtensionManager
3131
from .logging import disable_console_logging, enable_console_logging
3232
from .logging import enable_debug_logging_recommendation
@@ -288,6 +288,10 @@ async def _handle__worker_init_request(self, request):
288288
if not DependencyManager.is_in_linux_consumption():
289289
DependencyManager.prioritize_customer_dependencies()
290290

291+
if DependencyManager.is_in_linux_consumption() \
292+
and is_envvar_true(PYTHON_LOAD_FUNCTIONS_INIT):
293+
import azure.functions # NoQA
294+
291295
return protos.StreamingMessage(
292296
request_id=self.request_id,
293297
worker_init_response=protos.WorkerInitResponse(
@@ -450,6 +454,10 @@ async def _handle__invocation_request(self, request):
450454
shmem_mgr=self._shmem_mgr)
451455

452456
fi_context = self._get_context(invoc_request, fi.name, fi.directory)
457+
458+
# Use local thread storage to store the invocation ID
459+
# for a customer's threads
460+
fi_context.thread_local_storage.invocation_id = invocation_id
453461
if fi.requires_context:
454462
args['context'] = fi_context
455463

@@ -525,10 +533,11 @@ async def _handle__function_environment_reload_request(self, request):
525533
func_env_reload_request = \
526534
request.function_environment_reload_request
527535

528-
# Import before clearing path cache so that the default
529-
# azure.functions modules is available in sys.modules for
530-
# customer use
531-
import azure.functions # NoQA
536+
if not is_envvar_true(PYTHON_LOAD_FUNCTIONS_INIT):
537+
# Import before clearing path cache so that the default
538+
# azure.functions modules is available in sys.modules for
539+
# customer use
540+
import azure.functions # NoQA
532541

533542
# Append function project root to module finding sys.path
534543
if func_env_reload_request.function_app_directory:
@@ -660,7 +669,7 @@ def _get_context(invoc_request: protos.InvocationRequest, name: str,
660669

661670
return bindings.Context(
662671
name, directory, invoc_request.invocation_id,
663-
trace_context, retry_context)
672+
_invocation_id_local, trace_context, retry_context)
664673

665674
@disable_feature_by(constants.PYTHON_ROLLBACK_CWD_PATH)
666675
def _change_cwd(self, new_cwd: str):
@@ -728,12 +737,12 @@ def _create_sync_call_tp(
728737
def _run_sync_func(self, invocation_id, context, func, params):
729738
# This helper exists because we need to access the current
730739
# invocation_id from ThreadPoolExecutor's threads.
731-
_invocation_id_local.v = invocation_id
740+
context.thread_local_storage.invocation_id = invocation_id
732741
try:
733742
return ExtensionManager.get_sync_invocation_wrapper(context,
734743
func)(params)
735744
finally:
736-
_invocation_id_local.v = None
745+
context.thread_local_storage.invocation_id = None
737746

738747
async def _run_async_func(self, context, func, params):
739748
return await ExtensionManager.get_async_invocation_wrapper(
@@ -837,7 +846,7 @@ def get_current_invocation_id() -> Optional[str]:
837846
if task_invocation_id is not None:
838847
return task_invocation_id
839848

840-
return getattr(_invocation_id_local, 'v', None)
849+
return getattr(_invocation_id_local, 'invocation_id', None)
841850

842851

843852
_invocation_id_local = threading.local()

azure_functions_worker/version.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
33

4-
VERSION = '4.10.1'
4+
VERSION = '4.11.0'

python/prodV4/worker.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,13 @@ def determine_user_pkg_paths():
5353
func_worker_dir = str(Path(__file__).absolute().parent)
5454
env = os.environ
5555

56-
if is_azure_environment():
57-
user_pkg_paths = determine_user_pkg_paths()
58-
joined_pkg_paths = os.pathsep.join(user_pkg_paths)
56+
# Setting up python path for all environments to prioritize
57+
# third-party user packages over worker packages in PYTHONPATH
58+
user_pkg_paths = determine_user_pkg_paths()
59+
joined_pkg_paths = os.pathsep.join(user_pkg_paths)
60+
env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}'
5961

60-
# On cloud, we prioritize third-party user packages
61-
# over worker packages in PYTHONPATH
62-
env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}'
62+
if is_azure_environment():
6363
os.execve(sys.executable,
6464
[sys.executable, '-m', 'azure_functions_worker']
6565
+ sys.argv[1:],

0 commit comments

Comments
 (0)