From 02491452ff000d767933c0dbb8fc08447343715d Mon Sep 17 00:00:00 2001 From: David Justo Date: Sun, 9 Aug 2020 14:48:29 -0700 Subject: [PATCH 01/28] created helper classes --- .../durable_functions/DurableEntityContext.py | 77 +++++++++++++++++++ azure/durable_functions/entity.py | 50 ++++++++++++ .../models/DurableOrchestrationContext.py | 6 +- .../models/actions/ActionType.py | 1 + .../models/actions/CallEntityAction.py | 42 ++++++++++ .../models/entities/EntityState.py | 21 +++++ .../models/entities/OperationResult.py | 22 ++++++ .../models/entities/RequestMessage.py | 24 ++++++ .../models/entities/ResponseMessage.py | 4 + .../models/entities/Signal.py | 22 ++++++ .../models/entities/__init__.py | 12 +++ .../models/utils/entity_utils.py | 24 ++++++ azure/durable_functions/tasks/__init__.py | 2 + azure/durable_functions/tasks/call_entity.py | 52 +++++++++++++ .../durable_functions/tasks/task_utilities.py | 20 +++++ 15 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 azure/durable_functions/DurableEntityContext.py create mode 100644 azure/durable_functions/entity.py create mode 100644 azure/durable_functions/models/actions/CallEntityAction.py create mode 100644 azure/durable_functions/models/entities/EntityState.py create mode 100644 azure/durable_functions/models/entities/OperationResult.py create mode 100644 azure/durable_functions/models/entities/RequestMessage.py create mode 100644 azure/durable_functions/models/entities/ResponseMessage.py create mode 100644 azure/durable_functions/models/entities/Signal.py create mode 100644 azure/durable_functions/models/entities/__init__.py create mode 100644 azure/durable_functions/models/utils/entity_utils.py create mode 100644 azure/durable_functions/tasks/call_entity.py diff --git a/azure/durable_functions/DurableEntityContext.py b/azure/durable_functions/DurableEntityContext.py new file mode 100644 index 00000000..70ff31f6 --- /dev/null +++ b/azure/durable_functions/DurableEntityContext.py @@ -0,0 +1,77 @@ +from typing import Optional, Any +from .models.utils.entity_utils import EntityId +from .models.entities import EntityState, OperationResult +from azure.functions._durable_functions import _deserialize_custom_object +from datetime import datetime +import json + +class DurableEntityContext: + + def __init__(self, + entity_name: str, + entity_key: str, + entity_id: EntityId, + operation_name: Optional[str], + is_newly_constructed: bool): + self._entity_name: str = entity_name + self._entity_key: str = entity_key + self._operation_name: str = operation_name + self._is_newly_constructed: bool = is_newly_constructed + # TODO: the name _entity_state is very confusing + self._entity_state: EntityState = EntityState(results=[], signals=[]) + self.curr_req = None # TODO: we need to figure out this one, also better naming + + @property + def entity_name(self): + return self._entity_name + + @property + def entity_key(self): + return self._entity_key + + @property + def operation_name(self): + return self._operation_name + + @property + def is_newly_constructed(self): + return self._is_newly_constructed + + @classmethod + def from_json(cls, json_str: str): + json_dict = json.loads(json_str) + return cls(**json_dict) + + def set_state(self, state: Any) -> None: + self._entity_state.entity_exists = True + self._entity_state.entity_state = json.dumps(state) + + def get_input(self) -> Any: + input_ = None + req_input = self.curr_req.input + input_ = None if req_input is None else self.from_json_util(req_input) + return input_ + + def set_result(self, start_time: datetime, result: Any) -> None: + self._entity_state.entity_exists = True + # TODO: the helper below should be in a utils file + duration = self._elapsed_milliseconds_since(start_time) + new_result = OperationResult( + is_error=False, + duration=duration, + result=result) + self._entity_state.results.append(new_result) + + def destruct_on_exit(self) -> None: + self._entity_state.entity_exists = False + self._entity_state.entity_state = None + + def _elapsed_milliseconds_since(self, start_time: datetime): + end_time = datetime.now() + elapsed_time = end_time - start_time + # TODO: double-check this is a milliseconds diff + return elapsed_time + + def from_json_util(self, json_str): + # TODO: this should be a util elsewhere + return json.loads(json_str, object_hook=_deserialize_custom_object) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py new file mode 100644 index 00000000..b5ee3133 --- /dev/null +++ b/azure/durable_functions/entity.py @@ -0,0 +1,50 @@ +from models.entities import DurableEntityContext + +class Entity: + def __init__(self, entity_func): + self.fn = entity_func + + def handle(self, context): + raise ValueError("We did it!") + """ + ..... + return_state = ... + returnState.entity_exists = ... + return_state.entiy_state = ... + for(.............): + context ? + try: + # TODO: maybe move elapsed millis gen to the constructor of + # OperationResult + elapsed_millis = _elapsed_milliseconds_since(start_time) + operation_result =OperationResult( + is_error=False, + duration=elapsed_millis, + result=...... + + ) + ... + except Exception as e: + elapsed_millis = _elapsed_milliseconds_since(start_time) + operation_result = OperationResult( + is_error=True, + duration=elapsed_millis, + result=str(e) + ) + return_state.results.append(operation_result) + return entity_state.to_json_string() + """ + return None + + + + + @classmethod + def create(cls, fn): + def handle_(context) -> str: + # TODO: this requires some commenting, where do we need to get this from the body + context_body = getattr(context, "body", None) + if context_body is None: + context_body = context + return Entity(fn).handle(DurableEntityContext.from_json(context_body)) + return handle_ \ No newline at end of file diff --git a/azure/durable_functions/models/DurableOrchestrationContext.py b/azure/durable_functions/models/DurableOrchestrationContext.py index b2656c0a..03ef7ff0 100644 --- a/azure/durable_functions/models/DurableOrchestrationContext.py +++ b/azure/durable_functions/models/DurableOrchestrationContext.py @@ -10,7 +10,7 @@ from ..models.TokenSource import TokenSource from ..tasks import call_activity_task, task_all, task_any, call_activity_with_retry_task, \ wait_for_external_event_task, continue_as_new, new_uuid, call_http, create_timer_task, \ - call_sub_orchestrator_task, call_sub_orchestrator_with_retry_task + call_sub_orchestrator_task, call_sub_orchestrator_with_retry_task, call_entity_task from azure.functions._durable_functions import _deserialize_custom_object @@ -344,3 +344,7 @@ def function_context(self) -> FunctionContext: Object containing function level attributes not used by durable orchestrator. """ return self._function_context + + def call_entity(self, entityId: str, operationName: str, operationInput: Any): + return call_entity_task(entityId, operationName, operationInput) + diff --git a/azure/durable_functions/models/actions/ActionType.py b/azure/durable_functions/models/actions/ActionType.py index 8e42dbfe..66ffbb63 100644 --- a/azure/durable_functions/models/actions/ActionType.py +++ b/azure/durable_functions/models/actions/ActionType.py @@ -11,4 +11,5 @@ class ActionType(IntEnum): CONTINUE_AS_NEW: int = 4 CREATE_TIMER: int = 5 WAIT_FOR_EXTERNAL_EVENT: int = 6 + CALL_ENTITY = 7 CALL_HTTP: int = 8 diff --git a/azure/durable_functions/models/actions/CallEntityAction.py b/azure/durable_functions/models/actions/CallEntityAction.py new file mode 100644 index 00000000..e98fb3cf --- /dev/null +++ b/azure/durable_functions/models/actions/CallEntityAction.py @@ -0,0 +1,42 @@ +from typing import Any, Dict + +from .Action import Action +from .ActionType import ActionType +from ..utils.json_utils import add_attrib +from json import dumps +from azure.functions._durable_functions import _serialize_custom_object + + +class CallEntityAction(Action): + """Defines the structure of the Call Entity object. + + Provides the information needed by the durable extension to be able to call an activity + """ + + def __init__(self, entity_id: str, operation: str, input_=None): + self.entity_id: str = entity_id #TODO: type? + self.operation: str = operation + # It appears that `.input_` needs to be JSON-serializable at this point + self.input_ = dumps(input_, default=_serialize_custom_object) + + if not self.entity_id: + raise ValueError("entity_id cannot be empty") + + @property + def action_type(self) -> int: + """Get the type of action this class represents.""" + return ActionType.CALL_ENTITY + + def to_json(self) -> Dict[str, Any]: + """Convert object into a json dictionary. + + Returns + ------- + Dict[str, Any] + The instance of the class converted into a json dictionary + """ + json_dict = {} + add_attrib(json_dict, self, 'entity_id', 'entityId') + add_attrib(json_dict, self, 'operation', 'operation') + add_attrib(json_dict, self, 'input_', 'input') + return json_dict diff --git a/azure/durable_functions/models/entities/EntityState.py b/azure/durable_functions/models/entities/EntityState.py new file mode 100644 index 00000000..43086e54 --- /dev/null +++ b/azure/durable_functions/models/entities/EntityState.py @@ -0,0 +1,21 @@ +from typing import List, Optional +from . import OperationResult, Signal + +class EntityState: + def __init__(self, + results: List[OperationResult], + signals: List[Signal], + entity_exists: bool = False, + state: Optional[str] = None): + self.entity_exists = entity_exists + self.state = state + self._results = results + self._signals = signals + + @property + def results(self) -> List[OperationResult]: + return self._results + + @property + def signals(self) -> List[Signal]: + return self._signals \ No newline at end of file diff --git a/azure/durable_functions/models/entities/OperationResult.py b/azure/durable_functions/models/entities/OperationResult.py new file mode 100644 index 00000000..17f8f354 --- /dev/null +++ b/azure/durable_functions/models/entities/OperationResult.py @@ -0,0 +1,22 @@ +from typing import Optional + +class OperationResult: # inherit from orch state? + def __init__(self, + is_error: bool, + duration: int, + result: Optional[str] = None): + self._is_error: bool = is_error + self._duration: int = duration + self._result: Optional[str] = result + + @property + def is_error(self): + return self._is_error + + @property + def duration(self): + return self._duration + + @property + def result(self): + return self._result \ No newline at end of file diff --git a/azure/durable_functions/models/entities/RequestMessage.py b/azure/durable_functions/models/entities/RequestMessage.py new file mode 100644 index 00000000..f204848d --- /dev/null +++ b/azure/durable_functions/models/entities/RequestMessage.py @@ -0,0 +1,24 @@ +from typing import List, Optional, Any +from ..utils.entity_utils import EntityId + +class RequestMessage: + def __init__(self, + id_: str, + name: Optional[str] = None, + signal: Optional[bool] = None, + input_: Optional[str] = None, + arg: Optional[Any] = None, + parent: Optional[str] = None, + lockset: Optional[List[EntityId]] = None, + pos: Optional[int] = None): + # TODO: this class has too many optionals, may speak to + # over-caution, but it mimics the JS class. Investigate if + # these many Optionals are necessary. + self.id: id_ + self.name: name + self.signal: signal + self.input: input_ + self.arg: arg + self.parent: parent + self.lockset: lockset + self.pos: pos \ No newline at end of file diff --git a/azure/durable_functions/models/entities/ResponseMessage.py b/azure/durable_functions/models/entities/ResponseMessage.py new file mode 100644 index 00000000..b21dadce --- /dev/null +++ b/azure/durable_functions/models/entities/ResponseMessage.py @@ -0,0 +1,4 @@ +class ResponseMessage: + def __init__(self, result: str): + self.result = result + # TODO: JS has an additional exceptionType field, but does not use it \ No newline at end of file diff --git a/azure/durable_functions/models/entities/Signal.py b/azure/durable_functions/models/entities/Signal.py new file mode 100644 index 00000000..ec3f4edc --- /dev/null +++ b/azure/durable_functions/models/entities/Signal.py @@ -0,0 +1,22 @@ +from ..utils.entity_utils import EntityId + +class Signal: + def __init__(self, + target: EntityId, + name: str, + input_: str): + self._target = target + self._name = name + self._input = input_ + + @property + def target(self) -> EntityId: + return self._target + + @property + def name(self) -> str: + return self._name + + @property + def input(self) -> str: + return self._input \ No newline at end of file diff --git a/azure/durable_functions/models/entities/__init__.py b/azure/durable_functions/models/entities/__init__.py new file mode 100644 index 00000000..ddc96623 --- /dev/null +++ b/azure/durable_functions/models/entities/__init__.py @@ -0,0 +1,12 @@ +from .RequestMessage import RequestMessage +from .OperationResult import OperationResult +from .EntityState import EntityState +from .Signal import Signal +#TODO: this is very boilerplate-y, can we do better? + +__all__ = [ + 'RequestMessage', + 'OperationResult', + 'Signal', + 'EntityState' +] \ No newline at end of file diff --git a/azure/durable_functions/models/utils/entity_utils.py b/azure/durable_functions/models/utils/entity_utils.py new file mode 100644 index 00000000..b0e7e2f5 --- /dev/null +++ b/azure/durable_functions/models/utils/entity_utils.py @@ -0,0 +1,24 @@ +class EntityId: + @staticmethod + def get_scheduler_id(entity_id: EntityId) -> str: + return f"@{entity_id.name.lower()}@{entity_id.key}" + + @staticmethod + def get_entity_id(scheduler_id: str) -> EntityId: + sched_id_truncated = scheduler_id[1:] # we drop the starting `@` + components = sched_id_truncated.split("@") + if len(components) > 2: + raise ValueError("Unexpected format in SchedulerId") + [name, key] = components + return EntityId(name, key) + + def __init__(self, name: str, key: str): + if name == "": + raise ValueError("Entity name cannot be empty") + if key == "": + raise ValueError("Entity key cannot be empty") + self.name: str = name + self.key: str = key + + def __str__(self): + return EntityId.get_scheduler_id(entity_id=self) \ No newline at end of file diff --git a/azure/durable_functions/tasks/__init__.py b/azure/durable_functions/tasks/__init__.py index e91efe9f..82eb8be0 100644 --- a/azure/durable_functions/tasks/__init__.py +++ b/azure/durable_functions/tasks/__init__.py @@ -11,12 +11,14 @@ from .new_uuid import new_uuid from .call_http import call_http from .create_timer import create_timer_task +from .call_entity import call_entity_task __all__ = [ 'call_activity_task', 'call_activity_with_retry_task', 'call_sub_orchestrator_task', 'call_sub_orchestrator_with_retry_task', + 'call_entity_task', 'call_http', 'continue_as_new', 'new_uuid', diff --git a/azure/durable_functions/tasks/call_entity.py b/azure/durable_functions/tasks/call_entity.py new file mode 100644 index 00000000..9f775155 --- /dev/null +++ b/azure/durable_functions/tasks/call_entity.py @@ -0,0 +1,52 @@ +from typing import List, Any, Optional + +from ..models.Task import ( + Task) +from ..models.actions.CallEntityAction import CallEntityAction +from ..models.history import HistoryEvent, HistoryEventType +from .task_utilities import set_processed, parse_history_event, find_event +from ..models.utils.entity_utils import EntityId +from ..models.entities.RequestMessage import RequestMessage + + +def call_entity_task( + state: List[HistoryEvent], + entity_id: EntityId, + operation_name: str = "", + input_: Optional[Any] = None): + new_action = CallEntityAction(entity_id, operation_name, input_) + scheduler_id = EntityId.get_scheduler_id(entity_id=entity_id) + + hist_type = HistoryEventType.EVENT_SENT + extra_constraints = { + "InstanceId": scheduler_id, + "Name": "op" + } + event_sent = find_event(state, hist_type, extra_constraints) + + event_raised = None + if event_sent: + event_input = None + if hasattr(event_input, "Input"): + # TODO: we may need to create a superclass specialization constructor + event_input = RequestMessage(event_sent.Input) + hist_type = HistoryEventType.EVENT_RAISED + extra_constraints = { + "Name": event_input.id + } + event_raised = find_event(state, hist_type, extra_constraints) + # TODO: does it make sense to have an event_sent but no `Input` attribute ?? + # If not, we should raise an exception here + + set_processed([event_sent, event_raised]) + if event_raised is not None: + return Task( + is_completed=True, + is_faulted=False, + action=new_action, + result=parse_history_event(event_raised), + timestamp=event_raised.timestamp, + id_=event_raised.TaskScheduledId) + + # TODO: this may be missing exception handling, as is JS + return Task(is_completed=False, is_faulted=False, action=new_action) \ No newline at end of file diff --git a/azure/durable_functions/tasks/task_utilities.py b/azure/durable_functions/tasks/task_utilities.py index 167a31d5..81fd36f7 100644 --- a/azure/durable_functions/tasks/task_utilities.py +++ b/azure/durable_functions/tasks/task_utilities.py @@ -21,14 +21,34 @@ def parse_history_event(directive_result): # We provide the ability to deserialize custom objects, because the output of this # will be passed directly to the orchestrator as the output of some activity + # TODO: why do we have this chain of equivalent if-statements? if event_type == HistoryEventType.EVENT_RAISED: return json.loads(directive_result.Input, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.SUB_ORCHESTRATION_INSTANCE_COMPLETED: return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.TASK_COMPLETED: return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) + if event_type == HistoryEventType.EVENT_RAISED: + return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) return None +def find_event(state, event_type: HistoryEventType, extra_constraints: Dict[str, Any]): + def satisfies_contraints(e: HistoryEvent) -> bool: + for attr, val in extra_constraints: + if hasattr(e, attr) and getattr(e, attr) == val: + continue + else: + return False + return True + + tasks = [e for e in state + if e.event_type == event_type + and satisfies_contraints(e) and not e.is_processed] + + if len(tasks) == 0: + return None + + return tasks[0] def find_event_raised(state, name): """Find if the event with the given event name is raised. From e24f8a64730a60ab60115080b97c204b5b67b404 Mon Sep 17 00:00:00 2001 From: David Justo Date: Sun, 9 Aug 2020 15:13:34 -0700 Subject: [PATCH 02/28] typechecking entity-context --- azure/durable_functions/entity.py | 2 +- azure/durable_functions/models/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index b220f0d2..0f7e67e0 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -1,4 +1,4 @@ -from models.entities import DurableEntityContext +from .models import DurableEntityContext class Entity: def __init__(self, entity_func): diff --git a/azure/durable_functions/models/__init__.py b/azure/durable_functions/models/__init__.py index 8c9b6a39..e2e97748 100644 --- a/azure/durable_functions/models/__init__.py +++ b/azure/durable_functions/models/__init__.py @@ -10,11 +10,13 @@ from .TaskSet import TaskSet from .DurableHttpRequest import DurableHttpRequest from .TokenSource import ManagedIdentityTokenSource +from .DurableEntityContext import DurableEntityContext __all__ = [ 'DurableOrchestrationBindings', 'DurableOrchestrationClient', 'DurableOrchestrationContext', + 'DurableEntityContext', 'DurableHttpRequest', 'ManagedIdentityTokenSource', 'OrchestratorState', From 4f27d2b2e4baa466cc1a5bae0773060415d6644d Mon Sep 17 00:00:00 2001 From: David Justo Date: Sun, 9 Aug 2020 15:21:29 -0700 Subject: [PATCH 03/28] moving entity-context --- azure/durable_functions/{ => models}/DurableEntityContext.py | 4 ++-- azure/durable_functions/models/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename azure/durable_functions/{ => models}/DurableEntityContext.py (96%) diff --git a/azure/durable_functions/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py similarity index 96% rename from azure/durable_functions/DurableEntityContext.py rename to azure/durable_functions/models/DurableEntityContext.py index 70ff31f6..5ccb0cfe 100644 --- a/azure/durable_functions/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -1,6 +1,6 @@ from typing import Optional, Any -from .models.utils.entity_utils import EntityId -from .models.entities import EntityState, OperationResult +from .utils.entity_utils import EntityId +from .entities import EntityState, OperationResult from azure.functions._durable_functions import _deserialize_custom_object from datetime import datetime import json diff --git a/azure/durable_functions/models/__init__.py b/azure/durable_functions/models/__init__.py index e2e97748..cc291aa2 100644 --- a/azure/durable_functions/models/__init__.py +++ b/azure/durable_functions/models/__init__.py @@ -15,8 +15,8 @@ __all__ = [ 'DurableOrchestrationBindings', 'DurableOrchestrationClient', - 'DurableOrchestrationContext', 'DurableEntityContext', + 'DurableOrchestrationContext', 'DurableHttpRequest', 'ManagedIdentityTokenSource', 'OrchestratorState', From cf8d86a3a0b3f8abd5cd24f8e317354b06008032 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 24 Aug 2020 19:07:08 -0700 Subject: [PATCH 04/28] Durable entities are functional, linted, typechecked --- azure/durable_functions/__init__.py | 4 + azure/durable_functions/entity.py | 114 ++++++--- .../models/DurableEntityContext.py | 235 ++++++++++++++---- .../models/DurableOrchestrationContext.py | 22 +- .../models/actions/CallEntityAction.py | 14 +- .../models/entities/EntityState.py | 61 ++++- .../models/entities/OperationResult.py | 74 +++++- .../models/entities/RequestMessage.py | 47 +++- .../models/entities/ResponseMessage.py | 36 ++- .../models/entities/Signal.py | 44 +++- .../models/entities/__init__.py | 9 +- .../models/utils/entity_utils.py | 80 +++++- azure/durable_functions/tasks/call_entity.py | 37 ++- .../durable_functions/tasks/task_utilities.py | 43 +++- 14 files changed, 675 insertions(+), 145 deletions(-) diff --git a/azure/durable_functions/__init__.py b/azure/durable_functions/__init__.py index 8fb4f968..cf6c41fc 100644 --- a/azure/durable_functions/__init__.py +++ b/azure/durable_functions/__init__.py @@ -3,6 +3,8 @@ Exposes the different API components intended for public consumption """ from .orchestrator import Orchestrator +from .entity import Entity +from .models.utils.entity_utils import EntityId from .models.DurableOrchestrationClient import DurableOrchestrationClient from .models.DurableOrchestrationContext import DurableOrchestrationContext from .models.RetryOptions import RetryOptions @@ -10,6 +12,8 @@ __all__ = [ 'Orchestrator', + 'Entity', + 'EntityId', 'DurableOrchestrationClient', 'DurableOrchestrationContext', 'ManagedIdentityTokenSource', diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index 0f7e67e0..5134e23d 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -1,50 +1,92 @@ from .models import DurableEntityContext +from .models.entities import OperationResult, EntityState +from datetime import datetime +from typing import Callable, Any, List, Dict + class Entity: - def __init__(self, entity_func): - self.fn = entity_func - - def handle(self, context): - raise ValueError("We did it!") + """Durable Entity Class. + + Responsible for execuitng the user-defined entity function. + """ + + def __init__(self, entity_func: Callable[[DurableEntityContext], None]): + """Create a new entity for the user-defined entity. + + Responsible for executing the user-defined entity function + + Parameters + ---------- + entity_func: Callable[[DurableEntityContext], Generator[Any, Any, Any]] + The user defined entity function """ - ..... - return_state = ... - returnState.entity_exists = ... - return_state.entiy_state = ... - for(.............): - context ? - try: - # TODO: maybe move elapsed millis gen to the constructor of - # OperationResult - elapsed_millis = _elapsed_milliseconds_since(start_time) - operation_result =OperationResult( - is_error=False, - duration=elapsed_millis, - result=...... - - ) - ... - except Exception as e: - elapsed_millis = _elapsed_milliseconds_since(start_time) - operation_result = OperationResult( - is_error=True, - duration=elapsed_millis, - result=str(e) - ) - return_state.results.append(operation_result) - return entity_state.to_json_string() + self.fn: Callable[[DurableEntityContext], None] = entity_func + + def handle(self, context: DurableEntityContext, batch: List[Dict[str, Any]]) -> str: + """Handle the execution of the user-defined entity function. + + Loops over the batch, which serves to specify inputs to the entity, + and collects results and generates a final state, which are returned. + + Parameters + ---------- + context: DurableEntityContext + The entity context of the entity, which the user interacts with as their Durable API + + Returns + ------- + str + A JSON-formatted string representing the output state, results, and exceptions for the + entity execution. """ - return None + response = EntityState() + for packet in batch: + result: Any = None + is_error: bool = False + start_time: datetime = datetime.now() + + try: + # populate context + context._operation = packet["name"] + context._input = packet["input"] + self.fn(context) + result = context._result + except Exception as e: + is_error = True + result = str(e) + duration: int = context._elapsed_milliseconds_since(start_time) + operation_result = OperationResult( + is_error=is_error, + duration=duration, + result=result + ) + response.results.append(operation_result) + response.state = context._state + response.entity_exists = context._exists + return response.to_json_string() @classmethod - def create(cls, fn): - def handle_(context) -> str: + def create(cls, fn: Callable[[DurableEntityContext], None]) -> Callable[[Any], str]: + """Create an instance of the entity class. + + Parameters + ---------- + fn (Callable[[DurableEntityContext], None]): [description] + + Returns + ------- + Callable[[Any], str] + Handle function of the newly created entity client + """ + # TODO: review types here! + def handle(context) -> str: # TODO: this requires some commenting, where do we need to get this from the body context_body = getattr(context, "body", None) if context_body is None: context_body = context - return Entity(fn).handle(DurableEntityContext.from_json(context_body)) - return handle_ \ No newline at end of file + ctx, batch = DurableEntityContext.from_json(context_body) + return Entity(fn).handle(ctx, batch) + return handle diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index 5ccb0cfe..b80f63bb 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -1,77 +1,216 @@ -from typing import Optional, Any -from .utils.entity_utils import EntityId -from .entities import EntityState, OperationResult +from typing import Optional, Any, Dict, Tuple, List, Callable from azure.functions._durable_functions import _deserialize_custom_object from datetime import datetime import json + class DurableEntityContext: + """Context of the durable entity context. + + Describes the API used to specify durable entity user code. + """ def __init__(self, - entity_name: str, - entity_key: str, - entity_id: EntityId, - operation_name: Optional[str], - is_newly_constructed: bool): - self._entity_name: str = entity_name - self._entity_key: str = entity_key - self._operation_name: str = operation_name - self._is_newly_constructed: bool = is_newly_constructed - # TODO: the name _entity_state is very confusing - self._entity_state: EntityState = EntityState(results=[], signals=[]) - self.curr_req = None # TODO: we need to figure out this one, also better naming + name: str, + key: str, + exists: bool, + state: Any): + """Context of the durable entity context. + + Describes the API used to specify durable entity user code. + + Parameters + ---------- + name: str + The name of the Durable Entity + key: str + The key of the Durable Entity + exists: bool + Flag to determine if the entity exists + state: Any + The internal state of the Durable Entity + """ + self._entity_name: str = name + self._entity_key: str = key + + self._exists: bool = exists + self._is_newly_constructed: bool = False + + self._state: Any = state + self._input: Any = None + self._operation: Optional[str] = None + self._result: Any = None @property - def entity_name(self): + def entity_name(self) -> str: + """Get the name of the Entity. + + Returns + ------- + str + The name of the entity + """ return self._entity_name - + @property - def entity_key(self): + def entity_key(self) -> str: + """Get the Entity key. + + Returns + ------- + str + The entity key + """ return self._entity_key - + @property - def operation_name(self): - return self._operation_name - + def operation_name(self) -> Optional[str]: + """Get the current operation name. + + Returns + ------- + Optional[str] + The current operation name + """ + # TODO: Maybe we should raise an + # exception if _operation is None + return self._operation + @property - def is_newly_constructed(self): + def is_newly_constructed(self) -> bool: + """Determine if the Entity was newly constructed. + + Returns + ------- + bool + True if the Entity was newly constructed. False otherwise. + """ + # TODO: not updating this atm return self._is_newly_constructed @classmethod - def from_json(cls, json_str: str): + def from_json(cls, json_str: str) -> Tuple['DurableEntityContext', List[Dict[str, Any]]]: + """Instantiate a DurableEntityContext from a JSON-formatted string. + + Parameters + ---------- + json_string: str + A JSON-formatted string, returned by the durable-extension, + which represents the entity context + + Returns + ------- + DurableEntityContext + The DurableEntityContext originated from the input string + """ + # TODO: fix type json_dict = json.loads(json_str) - return cls(**json_dict) + json_dict["name"] = json_dict["self"]["name"] + json_dict["key"] = json_dict["self"]["key"] + json_dict.pop("self") + json_dict["state"] = json.loads(json_dict["state"], object_hook=_deserialize_custom_object) + batch = json_dict.pop("batch") + return cls(**json_dict), batch def set_state(self, state: Any) -> None: - self._entity_state.entity_exists = True - self._entity_state.entity_state = json.dumps(state) + """Set the state of the entity. + + Parameter + --------- + state: Any + The new state of the entity + """ + # TODO: enable serialization of custom types + self._exists = True + self._state = json.dumps(state) + + def get_state(self, initializer: Optional[Callable[[], Any]] = None) -> Any: + """Get the current state of this entity. + + Parameters + ---------- + initializer: Optional[Callable[[], Any]] + A 0-argument function to provide an initial state. Defaults to None. + + Returns + ------- + Any + The current state of the entity + """ + state = self._state + # TODO: some weird errs here with None states + if state is not None: + return state + elif initializer: + # TODO: ensure this is a fucntion + state = initializer() + return state def get_input(self) -> Any: + """Get the input for this operation. + + Returns + ------- + Any + The input for the current operation + """ input_ = None - req_input = self.curr_req.input + req_input = self._input + req_input = json.loads(req_input) input_ = None if req_input is None else self.from_json_util(req_input) return input_ - - def set_result(self, start_time: datetime, result: Any) -> None: - self._entity_state.entity_exists = True - # TODO: the helper below should be in a utils file - duration = self._elapsed_milliseconds_since(start_time) - new_result = OperationResult( - is_error=False, - duration=duration, - result=result) - self._entity_state.results.append(new_result) - + + def set_result(self, result: Any) -> None: + """Set the result (return value) of the entity. + + Paramaters + ---------- + result: Any + The result / return value for the entity + """ + self._exists = True + self._result = result + def destruct_on_exit(self) -> None: - self._entity_state.entity_exists = False - self._entity_state.entity_state = None - - def _elapsed_milliseconds_since(self, start_time: datetime): + """Delete this entity after the operation completes.""" + self._exists = False + self._state = None + + def _elapsed_milliseconds_since(self, start_time: datetime) -> int: + """Calculate the elapsed time, in milliseconds, from the start_time to the present. + + Parameters + ---------- + start_time: datetime + The timestamp of when the entity began processing a batched request. + + Returns + ------- + int + The time, in millseconds, from start_time to now + """ end_time = datetime.now() - elapsed_time = end_time - start_time - # TODO: double-check this is a milliseconds diff + time_diff = end_time - start_time + elapsed_time = int(time_diff.total_seconds() * 1000) return elapsed_time - - def from_json_util(self, json_str): - # TODO: this should be a util elsewhere + + def from_json_util(self, json_str: str) -> Any: + """Load an arbitrary datatype from its JSON representation. + + The Out-of-proc SDK has a special JSON encoding strategy + to enable arbitrary datatypes to be serialized. This utility + loads a JSON with the assumption that it follows that encoding + method. + + Parameters + ---------- + json_str: str + A JSON-formatted string, from durable-extension + + Returns + ------- + Any: + The original datatype that was serialized + """ + # TODO: this should be a util elsewhere, since we use it alot return json.loads(json_str, object_hook=_deserialize_custom_object) diff --git a/azure/durable_functions/models/DurableOrchestrationContext.py b/azure/durable_functions/models/DurableOrchestrationContext.py index 32628a9b..f2d96186 100644 --- a/azure/durable_functions/models/DurableOrchestrationContext.py +++ b/azure/durable_functions/models/DurableOrchestrationContext.py @@ -9,6 +9,7 @@ from .actions import Action from ..models.Task import Task from ..models.TokenSource import TokenSource +from .utils.entity_utils import EntityId from ..tasks import call_activity_task, task_all, task_any, call_activity_with_retry_task, \ wait_for_external_event_task, continue_as_new, new_uuid, call_http, create_timer_task, \ call_sub_orchestrator_task, call_sub_orchestrator_with_retry_task, call_entity_task @@ -359,8 +360,25 @@ def function_context(self) -> FunctionContext: """ return self._function_context - def call_entity(self, entityId: str, operationName: str, operationInput: Any): - return call_entity_task(entityId, operationName, operationInput) + def call_entity(self, entityId: EntityId, + operationName: str, operationInput: Optional[Any] = None): + """Get the result of Durable Entity operation given some input. + + Parameters + ---------- + entityId: EntityId + The ID of the entity to call + operationName: str + The operation to execute + operationInput: Optional[Any] + The input for tne operation, defaults to None. + + Returns + ------- + Task + A Task of the entity call + """ + return call_entity_task(self.histories, entityId, operationName, operationInput) @property def will_continue_as_new(self) -> bool: diff --git a/azure/durable_functions/models/actions/CallEntityAction.py b/azure/durable_functions/models/actions/CallEntityAction.py index e98fb3cf..d7d8d74d 100644 --- a/azure/durable_functions/models/actions/CallEntityAction.py +++ b/azure/durable_functions/models/actions/CallEntityAction.py @@ -5,6 +5,7 @@ from ..utils.json_utils import add_attrib from json import dumps from azure.functions._durable_functions import _serialize_custom_object +from ..utils.entity_utils import EntityId class CallEntityAction(Action): @@ -13,11 +14,11 @@ class CallEntityAction(Action): Provides the information needed by the durable extension to be able to call an activity """ - def __init__(self, entity_id: str, operation: str, input_=None): - self.entity_id: str = entity_id #TODO: type? + def __init__(self, entity_id: EntityId, operation: str, input_=None): + self.entity_id: EntityId = entity_id + self.instance_id: str = EntityId.get_scheduler_id(entity_id) self.operation: str = operation - # It appears that `.input_` needs to be JSON-serializable at this point - self.input_ = dumps(input_, default=_serialize_custom_object) + self.input_: str = dumps(input_, default=_serialize_custom_object) if not self.entity_id: raise ValueError("entity_id cannot be empty") @@ -35,8 +36,9 @@ def to_json(self) -> Dict[str, Any]: Dict[str, Any] The instance of the class converted into a json dictionary """ - json_dict = {} - add_attrib(json_dict, self, 'entity_id', 'entityId') + json_dict: Dict[str, Any] = {} + add_attrib(json_dict, self, "action_type", "actionType") + add_attrib(json_dict, self, 'instance_id', 'instanceId') add_attrib(json_dict, self, 'operation', 'operation') add_attrib(json_dict, self, 'input_', 'input') return json_dict diff --git a/azure/durable_functions/models/entities/EntityState.py b/azure/durable_functions/models/entities/EntityState.py index edecb0c3..7266d0aa 100644 --- a/azure/durable_functions/models/entities/EntityState.py +++ b/azure/durable_functions/models/entities/EntityState.py @@ -1,11 +1,18 @@ -from typing import List, Optional +from typing import List, Optional, Dict, Any from .Signal import Signal from .OperationResult import OperationResult +import json + class EntityState: + """Entity State. + + Used to communicate the state of the entity back to the durable extension + """ + def __init__(self, - results: List[OperationResult], - signals: List[Signal], + results: List[OperationResult] = [], + signals: List[Signal] = [], entity_exists: bool = False, state: Optional[str] = None): self.entity_exists = entity_exists @@ -15,8 +22,52 @@ def __init__(self, @property def results(self) -> List[OperationResult]: + """Get list of results of the entity. + + Returns + ------- + List[OperationResult]: + The results of the entity + """ return self._results - + @property def signals(self) -> List[Signal]: - return self._signals \ No newline at end of file + """Get list of signals to the entity. + + Returns + ------- + List[Signal]: + The signals of the entity + """ + return self._signals + + def to_json(self) -> Dict[str, Any]: + """Convert object into a json dictionary. + + Returns + ------- + Dict[str, Any] + The instance of the class converted into a json dictionary + """ + json_dict: Dict[str, Any] = {} + # Serialize the OperationResult list + serialized_results = list(map(lambda x: x.to_json(), self.results)) + + json_dict["entityExists"] = self.entity_exists + json_dict["entityState"] = self.state + json_dict["results"] = serialized_results + json_dict["signals"] = self.signals + return json_dict + + def to_json_string(self) -> str: + """Convert object into a json string. + + Returns + ------- + str + The instance of the object in json string format + """ + # TODO: Same implementation as in Orchestrator.py, we should refactor to shared a base + json_dict = self.to_json() + return json.dumps(json_dict) diff --git a/azure/durable_functions/models/entities/OperationResult.py b/azure/durable_functions/models/entities/OperationResult.py index 17f8f354..b3c65499 100644 --- a/azure/durable_functions/models/entities/OperationResult.py +++ b/azure/durable_functions/models/entities/OperationResult.py @@ -1,22 +1,76 @@ -from typing import Optional +from typing import Optional, Dict, Any + + +class OperationResult: + """OperationResult. + + The result of an Entity operation. + """ -class OperationResult: # inherit from orch state? def __init__(self, is_error: bool, duration: int, result: Optional[str] = None): + """Instantiate an OperationResult. + + Parameters + ---------- + is_error: bool + Whether or not the operation resulted in an exception. + duration: int + How long the operation took, in milliseconds. + result: Optional[str] + The operation result. Defaults to None. + """ + # TODO: perhaps this should inherit from orchestrator state self._is_error: bool = is_error - self._duration: int = duration + self._duration: int = duration self._result: Optional[str] = result - + @property - def is_error(self): + def is_error(self) -> bool: + """Determine if the operation resulted in an error. + + Returns + ------- + bool + True if the operation resulted in error. Otherwise False. + """ return self._is_error - + @property - def duration(self): + def duration(self) -> int: + """Get the duration of this operation. + + Returns + ------- + int: + The duration of this operation, in milliseconds + """ return self._duration - + @property - def result(self): - return self._result \ No newline at end of file + def result(self) -> Optional[str]: + """Get the operation's result. + + Returns + ------- + Optional[str] + The operation's result + """ + # TODO: is this necessary str or None? + return self._result + + def to_json(self) -> Dict[str, Any]: + """Represent OperationResult as a JSON-serializable Dict. + + Returns + ------- + Dict[str, Any] + A JSON-serializable Dict of the OperationResult + """ + to_json: Dict[str, Any] = {} + to_json["isError"] = self.is_error + to_json["duration"] = self.duration + to_json["result"] = self.result + return to_json diff --git a/azure/durable_functions/models/entities/RequestMessage.py b/azure/durable_functions/models/entities/RequestMessage.py index f204848d..0b82dd4d 100644 --- a/azure/durable_functions/models/entities/RequestMessage.py +++ b/azure/durable_functions/models/entities/RequestMessage.py @@ -1,7 +1,14 @@ from typing import List, Optional, Any from ..utils.entity_utils import EntityId +import json + class RequestMessage: + """RequestMessage. + + Specifies a request to an entity. + """ + def __init__(self, id_: str, name: Optional[str] = None, @@ -10,15 +17,37 @@ def __init__(self, arg: Optional[Any] = None, parent: Optional[str] = None, lockset: Optional[List[EntityId]] = None, - pos: Optional[int] = None): + pos: Optional[int] = None, + **kwargs): # TODO: this class has too many optionals, may speak to # over-caution, but it mimics the JS class. Investigate if # these many Optionals are necessary. - self.id: id_ - self.name: name - self.signal: signal - self.input: input_ - self.arg: arg - self.parent: parent - self.lockset: lockset - self.pos: pos \ No newline at end of file + self.id = id_ + self.name = name + self.signal = signal + self.input = input_ + self.arg = arg + self.parent = parent + self.lockset = lockset + self.pos = pos + + @classmethod + def from_json(cls, json_str: str) -> 'RequestMessage': + """Instantiate a RequestMessage object from the durable-extension provided JSON data. + + Parameters + ---------- + json_str: str + A durable-extension provided json-formatted string representation of + a RequestMessage + + Returns + ------- + RequestMessage: + A RequestMessage object from the json_str parameter + """ + # We replace the `id` key for `id_` to avoid clashes with reserved + # identifiers in Python + json_dict = json.loads(json_str) + json_dict["id_"] = json_dict.pop("id") + return cls(**json_dict) diff --git a/azure/durable_functions/models/entities/ResponseMessage.py b/azure/durable_functions/models/entities/ResponseMessage.py index b21dadce..ffd58985 100644 --- a/azure/durable_functions/models/entities/ResponseMessage.py +++ b/azure/durable_functions/models/entities/ResponseMessage.py @@ -1,4 +1,38 @@ +from typing import Dict, Any + + class ResponseMessage: + """ResponseMessage. + + Specifies the response of an entity, as processed by the durable-extension. + """ + def __init__(self, result: str): + """Instantiate a ResponseMessage. + + Specifies the response of an entity, as processed by the durable-extension. + + Parameters + ---------- + result: str + The result provided by the entity + """ self.result = result - # TODO: JS has an additional exceptionType field, but does not use it \ No newline at end of file + # TODO: JS has an additional exceptionType field, but does not use it + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> 'ResponseMessage': + """Instantiate a ResponseMessage from a dict of the JSON-response by the extension. + + Parameters + ---------- + d: Dict[str, Any] + The dictionary parsed from the JSON-response by the durable-extension + + Returns + ------- + ResponseMessage: + The ResponseMessage built from the provided dictionary + """ + result = cls(d["result"]) + return result diff --git a/azure/durable_functions/models/entities/Signal.py b/azure/durable_functions/models/entities/Signal.py index ec3f4edc..75a1c8df 100644 --- a/azure/durable_functions/models/entities/Signal.py +++ b/azure/durable_functions/models/entities/Signal.py @@ -1,22 +1,62 @@ from ..utils.entity_utils import EntityId + class Signal: + """An EntitySignal. + + Describes a signal call to a Durable Entity. + """ + def __init__(self, target: EntityId, name: str, input_: str): + """Instantiate an EntitySignal. + + Instantiate a signal call to a Durable Entity. + + Parameters + ---------- + target: EntityId + The target of signal + name: str + The name of the signal + input_: str + The signal's input + """ self._target = target self._name = name self._input = input_ @property def target(self) -> EntityId: + """Get the Signal's target entity. + + Returns + ------- + EntityId + EntityId of the target + """ return self._target - + @property def name(self) -> str: + """Get the Signal's name. + + Returns + ------- + str + The Signal's name + """ return self._name @property def input(self) -> str: - return self._input \ No newline at end of file + """Get the Signal's input. + + Returns + ------- + str + The Signal's input + """ + return self._input diff --git a/azure/durable_functions/models/entities/__init__.py b/azure/durable_functions/models/entities/__init__.py index ddc96623..24eeda7f 100644 --- a/azure/durable_functions/models/entities/__init__.py +++ b/azure/durable_functions/models/entities/__init__.py @@ -1,12 +1,17 @@ +"""Utility classes used by the Durable Function python library for dealing with entities. + +_Internal Only_ +""" + from .RequestMessage import RequestMessage from .OperationResult import OperationResult from .EntityState import EntityState from .Signal import Signal -#TODO: this is very boilerplate-y, can we do better? +# TODO: this is very boilerplate-y, can we do better? __all__ = [ 'RequestMessage', 'OperationResult', 'Signal', 'EntityState' -] \ No newline at end of file +] diff --git a/azure/durable_functions/models/utils/entity_utils.py b/azure/durable_functions/models/utils/entity_utils.py index 0f415ee5..c007aed0 100644 --- a/azure/durable_functions/models/utils/entity_utils.py +++ b/azure/durable_functions/models/utils/entity_utils.py @@ -1,24 +1,80 @@ class EntityId: + """EntityId. + + It identifies an entity by its name and its key. + """ + + def __init__(self, name: str, key: str): + """Instantiate an EntityId object. + + Identifies an entity by its name and its key. + + Parameters + ---------- + name: str + The entity name + key: str + The entity key + + Raises + ------ + ValueError: If the entity name or key are the empty string + """ + if name == "": + raise ValueError("Entity name cannot be empty") + if key == "": + raise ValueError("Entity key cannot be empty") + self.name: str = name + self.key: str = key + @staticmethod def get_scheduler_id(entity_id: 'EntityId') -> str: + """Produce a SchedulerId from an EntityId. + + Parameters + ---------- + entity_id: EntityId + An EntityId object + + Returns + ------- + str: + A SchedulerId representation of the input EntityId + """ return f"@{entity_id.name.lower()}@{entity_id.key}" @staticmethod def get_entity_id(scheduler_id: str) -> 'EntityId': - sched_id_truncated = scheduler_id[1:] # we drop the starting `@` + """Return an EntityId from a SchedulerId string. + + Parameters + ---------- + scheduler_id: str + The SchedulerId in which to base the returned EntityId + + Raises + ------ + ValueError: + When the SchedulerId string does not have the expected format + + Returns + ------- + EntityId: + An EntityId object based on the SchedulerId string + """ + sched_id_truncated = scheduler_id[1:] # we drop the starting `@` components = sched_id_truncated.split("@") if len(components) > 2: raise ValueError("Unexpected format in SchedulerId") [name, key] = components return EntityId(name, key) - - def __init__(self, name: str, key: str): - if name == "": - raise ValueError("Entity name cannot be empty") - if key == "": - raise ValueError("Entity key cannot be empty") - self.name: str = name - self.key: str = key - - def __str__(self): - return EntityId.get_scheduler_id(entity_id=self) \ No newline at end of file + + def __str__(self) -> str: + """Print the string representation of this EntityId. + + Returns + ------- + str: + A SchedulerId-based string representation of the EntityId + """ + return EntityId.get_scheduler_id(entity_id=self) diff --git a/azure/durable_functions/tasks/call_entity.py b/azure/durable_functions/tasks/call_entity.py index 9f775155..9d22989f 100644 --- a/azure/durable_functions/tasks/call_entity.py +++ b/azure/durable_functions/tasks/call_entity.py @@ -7,6 +7,7 @@ from .task_utilities import set_processed, parse_history_event, find_event from ..models.utils.entity_utils import EntityId from ..models.entities.RequestMessage import RequestMessage +from ..models.entities.ResponseMessage import ResponseMessage def call_entity_task( @@ -14,6 +15,28 @@ def call_entity_task( entity_id: EntityId, operation_name: str = "", input_: Optional[Any] = None): + """Determine the status of a call-entity task. + + It the task hasn't been scheduled, it returns a Task to schedule. If the task completed, + we return a completed Task, to process its result. + + Parameters + ---------- + state: List[HistoryEvent] + The list of history events to search over to determine the + current state of the callEntity Task. + entity_id: EntityId + An identifier for the entity to call. + operation_name: str + The name of the operation the entity needs to execute. + input_: Any + The JSON-serializable input to pass to the activity function. + + Returns + ------- + Task + A Durable Task that completes when the called entity completes or fails. + """ new_action = CallEntityAction(entity_id, operation_name, input_) scheduler_id = EntityId.get_scheduler_id(entity_id=entity_id) @@ -27,9 +50,8 @@ def call_entity_task( event_raised = None if event_sent: event_input = None - if hasattr(event_input, "Input"): - # TODO: we may need to create a superclass specialization constructor - event_input = RequestMessage(event_sent.Input) + if hasattr(event_sent, "Input"): + event_input = RequestMessage.from_json(event_sent.Input) hist_type = HistoryEventType.EVENT_RAISED extra_constraints = { "Name": event_input.id @@ -40,13 +62,16 @@ def call_entity_task( set_processed([event_sent, event_raised]) if event_raised is not None: + response = parse_history_event(event_raised) + response = ResponseMessage.from_dict(response) + result = response.result return Task( is_completed=True, is_faulted=False, action=new_action, - result=parse_history_event(event_raised), + result=result, timestamp=event_raised.timestamp, - id_=event_raised.TaskScheduledId) + id_=event_raised.Name) # event_raised.TaskScheduledId # TODO: this may be missing exception handling, as is JS - return Task(is_completed=False, is_faulted=False, action=new_action) \ No newline at end of file + return Task(is_completed=False, is_faulted=False, action=new_action) diff --git a/azure/durable_functions/tasks/task_utilities.py b/azure/durable_functions/tasks/task_utilities.py index 53020bb2..ccc8d6c0 100644 --- a/azure/durable_functions/tasks/task_utilities.py +++ b/azure/durable_functions/tasks/task_utilities.py @@ -22,23 +22,53 @@ def parse_history_event(directive_result): # We provide the ability to deserialize custom objects, because the output of this # will be passed directly to the orchestrator as the output of some activity # TODO: why do we have this chain of equivalent if-statements? - if event_type == HistoryEventType.EVENT_RAISED: - return json.loads(directive_result.Input, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.SUB_ORCHESTRATION_INSTANCE_COMPLETED: return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.TASK_COMPLETED: return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.EVENT_RAISED: - return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) + # TODO: why is the result here on input? + return json.loads(directive_result.Input, object_hook=_deserialize_custom_object) return None -def find_event(state, event_type: HistoryEventType, extra_constraints: Dict[str, Any]): + +def find_event(state: List[HistoryEvent], event_type: HistoryEventType, + extra_constraints: Dict[str, Any]) -> Optional[HistoryEvent]: + """Find event in the histories array as per some constraints. + + Parameters + ---------- + state: List[HistoryEvent] + The list of events so far in the orchestaration + event_type: HistoryEventType + The type of the event we're looking for + extra_constraints: Dict[str, Any] + A dictionary of key-value pairs where the key is a property of the + sought-after event, and value are its expected contents. + + Returns + ------- + Optional[HistoryEvent] + The event being searched-for, if found. Else, None. + """ def satisfies_contraints(e: HistoryEvent) -> bool: - for attr, val in extra_constraints: + """Determine if an event matches our search criteria. + + Parameters + ---------- + e: HistoryEvent + An event from the state array + + Returns + ------- + bool + True if the event matches our constraints. Else, False. + """ + for attr, val in extra_constraints.items(): if hasattr(e, attr) and getattr(e, attr) == val: continue else: - return False + return False return True tasks = [e for e in state @@ -50,6 +80,7 @@ def satisfies_contraints(e: HistoryEvent) -> bool: return tasks[0] + def find_event_raised(state, name): """Find if the event with the given event name is raised. From 402b9ad39750b23de9406bf7b06f88db005aab16 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 24 Aug 2020 23:02:58 -0700 Subject: [PATCH 05/28] added simple test --- tests/test_utils/ContextBuilder.py | 14 ++++++++++++-- tests/test_utils/json_utils.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_utils/ContextBuilder.py b/tests/test_utils/ContextBuilder.py index 753009fa..6ec1f76f 100644 --- a/tests/test_utils/ContextBuilder.py +++ b/tests/test_utils/ContextBuilder.py @@ -63,6 +63,13 @@ def add_sub_orchestrator_failed_event(self, id_, reason, details): event.TaskScheduledId = id_ self.history_events.append(event) + def add_event_sent_event(self, instance_id): + event = self.get_base_event(HistoryEventType.EVENT_SENT) + event.InstanceId = instance_id + event.Name = "op" + event.Input = json.dumps({ "id": "0000" }) # usually provided by the extension + self.history_events.append(event) + def add_task_scheduled_event( self, name: str, id_: int, version: str = '', input_=None): event = self.get_base_event(HistoryEventType.TASK_SCHEDULED, id_=id_) @@ -109,10 +116,13 @@ def add_execution_started_event( event.Input = input_ self.history_events.append(event) - def add_event_raised_event(self, name: str, id_: int, input_=None, timestamp=None): + def add_event_raised_event(self, name:str, id_: int, input_=None, timestamp=None, is_entity=False): event = self.get_base_event(HistoryEventType.EVENT_RAISED, id_=id_, timestamp=timestamp) event.Name = name - event.Input = input_ + if is_entity: + event.Input = json.dumps({ "result": input_ }) + else: + event.Input = input_ # event.timestamp = timestamp self.history_events.append(event) diff --git a/tests/test_utils/json_utils.py b/tests/test_utils/json_utils.py index 834ada18..03eaf6aa 100644 --- a/tests/test_utils/json_utils.py +++ b/tests/test_utils/json_utils.py @@ -24,7 +24,7 @@ def convert_history_event_to_json_dict( add_attrib(json_dict, history_event, 'FireAt') add_attrib(json_dict, history_event, 'TimerId') add_attrib(json_dict, history_event, 'Name') + add_attrib(json_dict, history_event, 'InstanceId') add_json_attrib(json_dict, history_event, 'orchestration_instance', 'OrchestrationInstance') - return json_dict From 14844b36960c509bc4a7e48312e562fc589c121a Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 24 Aug 2020 23:20:31 -0700 Subject: [PATCH 06/28] added sample, missing test --- azure/durable_functions/__init__.py | 2 + samples/counter_entity/.funcignore | 1 + samples/counter_entity/.gitignore | 132 ++++++++++++++++++ samples/counter_entity/Counter/__init__.py | 36 +++++ samples/counter_entity/Counter/function.json | 10 ++ .../DurableOrchestration/__init__.py | 42 ++++++ .../DurableOrchestration/function.json | 11 ++ .../counter_entity/DurableTrigger/__init__.py | 30 ++++ .../DurableTrigger/function.json | 27 ++++ samples/counter_entity/README.md | 35 +++++ samples/counter_entity/host.json | 3 + samples/counter_entity/local.settings.json | 7 + samples/counter_entity/requirements.txt | 2 + tests/orchestrator/test_entity.py | 71 ++++++++++ 14 files changed, 409 insertions(+) create mode 100644 samples/counter_entity/.funcignore create mode 100644 samples/counter_entity/.gitignore create mode 100644 samples/counter_entity/Counter/__init__.py create mode 100644 samples/counter_entity/Counter/function.json create mode 100644 samples/counter_entity/DurableOrchestration/__init__.py create mode 100644 samples/counter_entity/DurableOrchestration/function.json create mode 100644 samples/counter_entity/DurableTrigger/__init__.py create mode 100644 samples/counter_entity/DurableTrigger/function.json create mode 100644 samples/counter_entity/README.md create mode 100644 samples/counter_entity/host.json create mode 100644 samples/counter_entity/local.settings.json create mode 100644 samples/counter_entity/requirements.txt create mode 100644 tests/orchestrator/test_entity.py diff --git a/azure/durable_functions/__init__.py b/azure/durable_functions/__init__.py index cf6c41fc..d950dde9 100644 --- a/azure/durable_functions/__init__.py +++ b/azure/durable_functions/__init__.py @@ -7,6 +7,7 @@ from .models.utils.entity_utils import EntityId from .models.DurableOrchestrationClient import DurableOrchestrationClient from .models.DurableOrchestrationContext import DurableOrchestrationContext +from .models.DurableEntityContext import DurableEntityContext from .models.RetryOptions import RetryOptions from .models.TokenSource import ManagedIdentityTokenSource @@ -15,6 +16,7 @@ 'Entity', 'EntityId', 'DurableOrchestrationClient', + 'DurableEntityContext', 'DurableOrchestrationContext', 'ManagedIdentityTokenSource', 'RetryOptions' diff --git a/samples/counter_entity/.funcignore b/samples/counter_entity/.funcignore new file mode 100644 index 00000000..f5e96dbf --- /dev/null +++ b/samples/counter_entity/.funcignore @@ -0,0 +1 @@ +venv \ No newline at end of file diff --git a/samples/counter_entity/.gitignore b/samples/counter_entity/.gitignore new file mode 100644 index 00000000..4e426b7a --- /dev/null +++ b/samples/counter_entity/.gitignore @@ -0,0 +1,132 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don’t work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Azure Functions artifacts +bin +obj +appsettings.json +.python_packages + +# pycharm +.idea diff --git a/samples/counter_entity/Counter/__init__.py b/samples/counter_entity/Counter/__init__.py new file mode 100644 index 00000000..821e1bd4 --- /dev/null +++ b/samples/counter_entity/Counter/__init__.py @@ -0,0 +1,36 @@ +import logging +import json + +import azure.functions as func +import azure.durable_functions as df + + +def entity_function(context: df.DurableEntityContext): + """A Counter Durable Entity. + + A simple example of a Durable Entity that implements + a simple counter. + + Parameters + ---------- + context (df.DurableEntityContext): + The Durable Entity context, which exports an API + for implementing durable entities. + """ + + current_value = context.get_state(lambda: "0") + operation = context.operation_name + if operation == "add": + amount = context.get_input() + current_value += amount + elif operation == "reset": + current_value = 0 + elif operation == "get": + pass + + #TODO: we need to handle an invalid op, throw exception? + context.set_state(current_value) + context.set_result(current_value) + + +main = df.Entity.create(entity_function) \ No newline at end of file diff --git a/samples/counter_entity/Counter/function.json b/samples/counter_entity/Counter/function.json new file mode 100644 index 00000000..c5d7d9ed --- /dev/null +++ b/samples/counter_entity/Counter/function.json @@ -0,0 +1,10 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "name": "context", + "type": "entityTrigger", + "direction": "in" + } + ] +} diff --git a/samples/counter_entity/DurableOrchestration/__init__.py b/samples/counter_entity/DurableOrchestration/__init__.py new file mode 100644 index 00000000..11ddfe5b --- /dev/null +++ b/samples/counter_entity/DurableOrchestration/__init__.py @@ -0,0 +1,42 @@ +# This function is not intended to be invoked directly. Instead it will be +# triggered by an HTTP starter function. +# Before running this sample, please: +# - create a Durable activity function (default name is "Hello") +# - create a Durable HTTP starter function +# - add azure-functions-durable to requirements.txt +# - run pip install -r requirements.txt + +import logging +import json + +import azure.functions as func +import azure.durable_functions as df + + +def orchestrator_function(context: df.DurableOrchestrationContext): + """This function provides the a simple implementation of an orchestrator + that calls a counter Durable Entity. + + Parameters + ---------- + context: DurableOrchestrationContext + This context has the past history and the durable orchestration API + + Returns + ------- + state + The state after applying the operation on the Durable Entity + + Yields + ------- + call_entity: Any + Yields to wait on the Durable Entity to process its operation, + with the provided input, to then receive its counter state value. + """ + + # TODO: make the operation and the input into URL params + entityId = df.EntityId("Counter", "myCounter") + state = yield context.call_entity(entityId, "add", 3) + return state + +main = df.Orchestrator.create(orchestrator_function) \ No newline at end of file diff --git a/samples/counter_entity/DurableOrchestration/function.json b/samples/counter_entity/DurableOrchestration/function.json new file mode 100644 index 00000000..46a44c50 --- /dev/null +++ b/samples/counter_entity/DurableOrchestration/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "name": "context", + "type": "orchestrationTrigger", + "direction": "in" + } + ], + "disabled": false +} diff --git a/samples/counter_entity/DurableTrigger/__init__.py b/samples/counter_entity/DurableTrigger/__init__.py new file mode 100644 index 00000000..0f11ca7b --- /dev/null +++ b/samples/counter_entity/DurableTrigger/__init__.py @@ -0,0 +1,30 @@ +import logging + +from azure.durable_functions import DurableOrchestrationClient +import azure.functions as func + + +async def main(req: func.HttpRequest, starter: str, message): + """This function starts up the orchestrator from an HTTP endpoint + + starter: str + A JSON-formatted string describing the orchestration context + + message: + An azure functions http output binding, it enables us to establish + an http response. + + Parameters + ---------- + req: func.HttpRequest + An HTTP Request object, it can be used to parse URL + parameters. + """ + + + function_name = req.route_params.get('functionName') + logging.info(starter) + client = DurableOrchestrationClient(starter) + instance_id = await client.start_new(function_name) + response = client.create_check_status_response(req, instance_id) + message.set(response) diff --git a/samples/counter_entity/DurableTrigger/function.json b/samples/counter_entity/DurableTrigger/function.json new file mode 100644 index 00000000..606d8d7c --- /dev/null +++ b/samples/counter_entity/DurableTrigger/function.json @@ -0,0 +1,27 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "name": "req", + "type": "httpTrigger", + "direction": "in", + "route": "orchestrators/{functionName}", + "methods": [ + "post", + "get" + ] + }, + { + "direction": "out", + "name": "message", + "type": "http" + }, + { + "name": "starter", + "type": "durableClient", + "direction": "in", + "datatype": "string" + } + ] +} \ No newline at end of file diff --git a/samples/counter_entity/README.md b/samples/counter_entity/README.md new file mode 100644 index 00000000..26b4ecea --- /dev/null +++ b/samples/counter_entity/README.md @@ -0,0 +1,35 @@ +# Durable Entities - Sample + +This sample exemplifies how to go about using the [Durable Entities](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-entities?tabs=csharp) construct in Python Durable Functions. + +## Usage Instructions + +### Create a `local.settings.json` file in this directory +This file stores app settings, connection strings, and other settings used by local development tools. Learn more about it [here](https://docs.microsoft.com/en-us/azure/azure-functions/functions-run-local?tabs=windows%2Ccsharp%2Cbash#local-settings-file). +For this sample, you will only need an `AzureWebJobsStorage` connection string, which you can obtain from the Azure portal. + +With you connection string, your `local.settings.json` file should look as follows, with `` replaced with the connection string you obtained from the Azure portal: + +```json +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "", + "FUNCTIONS_WORKER_RUNTIME": "python" + } +} +``` + +### Run the Sample +To try this sample, run `func host start` in this directory. If all the system requirements have been met, and +after some initialization logs, you should see something like the following: + +```bash +Http Functions: + + DurableTrigger: [POST,GET] http://localhost:7071/api/orchestrators/{functionName} +``` + +This indicates that your `DurableTrigger` function can be reached via a `GET` or `POST` request to that URL. `DurableTrigger` starts the function-chaning orchestrator whose name is passed as a parameter to the URL. So, to start the orchestrator, which is named `DurableOrchestration`, make a GET request to `http://127.0.0.1:7071/api/orchestrators/DurableOrchestration`. + +And that's it! You should see a JSON response with five URLs to monitor the status of the orchestration. \ No newline at end of file diff --git a/samples/counter_entity/host.json b/samples/counter_entity/host.json new file mode 100644 index 00000000..81e35b7b --- /dev/null +++ b/samples/counter_entity/host.json @@ -0,0 +1,3 @@ +{ + "version": "2.0" +} \ No newline at end of file diff --git a/samples/counter_entity/local.settings.json b/samples/counter_entity/local.settings.json new file mode 100644 index 00000000..a2ded917 --- /dev/null +++ b/samples/counter_entity/local.settings.json @@ -0,0 +1,7 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "python" + } +} diff --git a/samples/counter_entity/requirements.txt b/samples/counter_entity/requirements.txt new file mode 100644 index 00000000..e8934e6e --- /dev/null +++ b/samples/counter_entity/requirements.txt @@ -0,0 +1,2 @@ +azure-functions +#azure-functions-durable>=1.0.0b6 \ No newline at end of file diff --git a/tests/orchestrator/test_entity.py b/tests/orchestrator/test_entity.py new file mode 100644 index 00000000..5b33565e --- /dev/null +++ b/tests/orchestrator/test_entity.py @@ -0,0 +1,71 @@ +from .orchestrator_test_utils \ + import assert_orchestration_state_equals, get_orchestration_state_result, assert_valid_schema +from tests.test_utils.ContextBuilder import ContextBuilder +from azure.durable_functions.models.OrchestratorState import OrchestratorState +from azure.durable_functions.models.actions.CallEntityAction \ + import CallEntityAction +from tests.test_utils.testClasses import SerializableClass +import azure.durable_functions as df +from typing import Any + +def generator_function(context): + outputs = [] + entityId = df.EntityId("Counter", "myCounter") + x = yield context.call_entity(entityId, "add", 3) + + outputs.append(x) + return outputs + + +def base_expected_state(output=None) -> OrchestratorState: + return OrchestratorState(is_done=False, actions=[], output=output) + + +def add_call_entity_action(state: OrchestratorState, id_: df.EntityId, op: str, input_: Any): + action = CallEntityAction(entity_id=id_, operation=op, input_=input_) + state.actions.append([action]) + + +def add_call_entity_completed_events( + context_builder: ContextBuilder, op: str, instance_id=str, input_=None): + context_builder.add_event_sent_event(instance_id) + context_builder.add_orchestrator_completed_event() + context_builder.add_orchestrator_started_event() + context_builder.add_event_raised_event(name="0000", id_=0, input_=input_, is_entity=True) + print(dir(context_builder.history_events[2])) + + +def test_call_entity_sent(): + context_builder = ContextBuilder('test_simple_function') + + entityId = df.EntityId("Counter", "myCounter") + result = get_orchestration_state_result( + context_builder, generator_function) + + expected_state = base_expected_state() + add_call_entity_action(expected_state, entityId, "add", 3) + expected = expected_state.to_json() + + #assert_valid_schema(result) + assert_orchestration_state_equals(expected, result) + +def test_call_entity_raised(): + entityId = df.EntityId("Counter", "myCounter") + context_builder = ContextBuilder('test_simple_function') + add_call_entity_completed_events(context_builder, "add", df.EntityId.get_scheduler_id(entityId),3) + + print(dir(context_builder.history_events[2]), "LL") + result = get_orchestration_state_result( + context_builder, generator_function) + + expected_state = base_expected_state( + [3] + ) + + add_call_entity_action(expected_state, entityId, "add", 3) + expected_state._is_done = True + expected = expected_state.to_json() + + #assert_valid_schema(result) + + assert_orchestration_state_equals(expected, result) \ No newline at end of file From e69b2b5558499f93fc39cc4df2575791b46362d0 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 24 Aug 2020 23:24:19 -0700 Subject: [PATCH 07/28] removed leftover print statement --- tests/orchestrator/test_entity.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/orchestrator/test_entity.py b/tests/orchestrator/test_entity.py index 5b33565e..9878e1ea 100644 --- a/tests/orchestrator/test_entity.py +++ b/tests/orchestrator/test_entity.py @@ -32,8 +32,6 @@ def add_call_entity_completed_events( context_builder.add_orchestrator_completed_event() context_builder.add_orchestrator_started_event() context_builder.add_event_raised_event(name="0000", id_=0, input_=input_, is_entity=True) - print(dir(context_builder.history_events[2])) - def test_call_entity_sent(): context_builder = ContextBuilder('test_simple_function') @@ -54,7 +52,6 @@ def test_call_entity_raised(): context_builder = ContextBuilder('test_simple_function') add_call_entity_completed_events(context_builder, "add", df.EntityId.get_scheduler_id(entityId),3) - print(dir(context_builder.history_events[2]), "LL") result = get_orchestration_state_result( context_builder, generator_function) From 98db559d172e215fe259b65a9abaae0e513f5030 Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 26 Aug 2020 19:39:15 -0700 Subject: [PATCH 08/28] hidden object references are removed --- azure/durable_functions/entity.py | 2 +- azure/durable_functions/models/DurableEntityContext.py | 7 +++++-- azure/durable_functions/models/entities/EntityState.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index 5134e23d..0a635c28 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -39,7 +39,7 @@ def handle(self, context: DurableEntityContext, batch: List[Dict[str, Any]]) -> A JSON-formatted string representing the output state, results, and exceptions for the entity execution. """ - response = EntityState() + response = EntityState(results=[], signals=[]) for packet in batch: result: Any = None is_error: bool = False diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index b80f63bb..914be876 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -103,12 +103,15 @@ def from_json(cls, json_str: str) -> Tuple['DurableEntityContext', List[Dict[str DurableEntityContext The DurableEntityContext originated from the input string """ - # TODO: fix type json_dict = json.loads(json_str) json_dict["name"] = json_dict["self"]["name"] json_dict["key"] = json_dict["self"]["key"] json_dict.pop("self") - json_dict["state"] = json.loads(json_dict["state"], object_hook=_deserialize_custom_object) + + serialized_state = json_dict["state"] + if serialized_state is not None: + json_dict["state"] = json.loads(serialized_state, + object_hook=_deserialize_custom_object) batch = json_dict.pop("batch") return cls(**json_dict), batch diff --git a/azure/durable_functions/models/entities/EntityState.py b/azure/durable_functions/models/entities/EntityState.py index 7266d0aa..59fea5ad 100644 --- a/azure/durable_functions/models/entities/EntityState.py +++ b/azure/durable_functions/models/entities/EntityState.py @@ -11,8 +11,8 @@ class EntityState: """ def __init__(self, - results: List[OperationResult] = [], - signals: List[Signal] = [], + results: List[OperationResult], + signals: List[Signal], entity_exists: bool = False, state: Optional[str] = None): self.entity_exists = entity_exists From 0247d323d3b4a6b7e1acc257c27d9661642f34fb Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 11 Sep 2020 16:35:36 -0700 Subject: [PATCH 09/28] PR feedback --- azure/durable_functions/entity.py | 20 ++++++++++++++++++- .../models/DurableEntityContext.py | 18 ----------------- .../models/actions/CallEntityAction.py | 7 ++++--- .../models/utils/entity_utils.py | 2 +- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index 0a635c28..286361d0 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -56,7 +56,7 @@ def handle(self, context: DurableEntityContext, batch: List[Dict[str, Any]]) -> is_error = True result = str(e) - duration: int = context._elapsed_milliseconds_since(start_time) + duration: int = self._elapsed_milliseconds_since(start_time) operation_result = OperationResult( is_error=is_error, duration=duration, @@ -90,3 +90,21 @@ def handle(context) -> str: ctx, batch = DurableEntityContext.from_json(context_body) return Entity(fn).handle(ctx, batch) return handle + + def _elapsed_milliseconds_since(self, start_time: datetime) -> int: + """Calculate the elapsed time, in milliseconds, from the start_time to the present. + + Parameters + ---------- + start_time: datetime + The timestamp of when the entity began processing a batched request. + + Returns + ------- + int + The time, in millseconds, from start_time to now + """ + end_time = datetime.now() + time_diff = end_time - start_time + elapsed_time = int(time_diff.total_seconds() * 1000) + return elapsed_time diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index 914be876..52b852b5 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -179,24 +179,6 @@ def destruct_on_exit(self) -> None: self._exists = False self._state = None - def _elapsed_milliseconds_since(self, start_time: datetime) -> int: - """Calculate the elapsed time, in milliseconds, from the start_time to the present. - - Parameters - ---------- - start_time: datetime - The timestamp of when the entity began processing a batched request. - - Returns - ------- - int - The time, in millseconds, from start_time to now - """ - end_time = datetime.now() - time_diff = end_time - start_time - elapsed_time = int(time_diff.total_seconds() * 1000) - return elapsed_time - def from_json_util(self, json_str: str) -> Any: """Load an arbitrary datatype from its JSON representation. diff --git a/azure/durable_functions/models/actions/CallEntityAction.py b/azure/durable_functions/models/actions/CallEntityAction.py index d7d8d74d..4a546b94 100644 --- a/azure/durable_functions/models/actions/CallEntityAction.py +++ b/azure/durable_functions/models/actions/CallEntityAction.py @@ -16,13 +16,14 @@ class CallEntityAction(Action): def __init__(self, entity_id: EntityId, operation: str, input_=None): self.entity_id: EntityId = entity_id - self.instance_id: str = EntityId.get_scheduler_id(entity_id) - self.operation: str = operation - self.input_: str = dumps(input_, default=_serialize_custom_object) if not self.entity_id: raise ValueError("entity_id cannot be empty") + self.instance_id: str = EntityId.get_scheduler_id(entity_id) + self.operation: str = operation + self.input_: str = dumps(input_, default=_serialize_custom_object) + @property def action_type(self) -> int: """Get the type of action this class represents.""" diff --git a/azure/durable_functions/models/utils/entity_utils.py b/azure/durable_functions/models/utils/entity_utils.py index c007aed0..0ff33dd4 100644 --- a/azure/durable_functions/models/utils/entity_utils.py +++ b/azure/durable_functions/models/utils/entity_utils.py @@ -64,7 +64,7 @@ def get_entity_id(scheduler_id: str) -> 'EntityId': """ sched_id_truncated = scheduler_id[1:] # we drop the starting `@` components = sched_id_truncated.split("@") - if len(components) > 2: + if len(components) != 2: raise ValueError("Unexpected format in SchedulerId") [name, key] = components return EntityId(name, key) From b5135354e1dc145481ad9f21c2d041db5d0caa1b Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 11 Sep 2020 16:45:17 -0700 Subject: [PATCH 10/28] minor change in CallEntityAction to overcomposate for GitHub acting up --- azure/durable_functions/models/actions/CallEntityAction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/azure/durable_functions/models/actions/CallEntityAction.py b/azure/durable_functions/models/actions/CallEntityAction.py index 4a546b94..55baa4ef 100644 --- a/azure/durable_functions/models/actions/CallEntityAction.py +++ b/azure/durable_functions/models/actions/CallEntityAction.py @@ -17,6 +17,7 @@ class CallEntityAction(Action): def __init__(self, entity_id: EntityId, operation: str, input_=None): self.entity_id: EntityId = entity_id + # Validating that EntityId exists before trying to parse its instanceId if not self.entity_id: raise ValueError("entity_id cannot be empty") From 023fd70b7e099f0da2eb24bf13ad834cc54fec61 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 11 Sep 2020 16:54:09 -0700 Subject: [PATCH 11/28] linting --- azure/durable_functions/models/DurableEntityContext.py | 1 - azure/durable_functions/tasks/task_utilities.py | 1 - 2 files changed, 2 deletions(-) diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index 52b852b5..c4c13879 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -1,6 +1,5 @@ from typing import Optional, Any, Dict, Tuple, List, Callable from azure.functions._durable_functions import _deserialize_custom_object -from datetime import datetime import json diff --git a/azure/durable_functions/tasks/task_utilities.py b/azure/durable_functions/tasks/task_utilities.py index a21cb64e..9cd1aadd 100644 --- a/azure/durable_functions/tasks/task_utilities.py +++ b/azure/durable_functions/tasks/task_utilities.py @@ -8,7 +8,6 @@ from ..models.Task import Task - def should_suspend(partial_result) -> bool: """Check the state of the result to determine if the orchestration should suspend.""" return bool(partial_result is not None From 65188f28fc2f6951a91293435bd6d51cef6a193a Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 14 Sep 2020 09:51:33 -0700 Subject: [PATCH 12/28] Rewind API (#163) --- .../models/DurableOrchestrationClient.py | 54 +++++++++++++++++++ .../models/test_DurableOrchestrationClient.py | 52 ++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/azure/durable_functions/models/DurableOrchestrationClient.py b/azure/durable_functions/models/DurableOrchestrationClient.py index 1442124a..a94f5722 100644 --- a/azure/durable_functions/models/DurableOrchestrationClient.py +++ b/azure/durable_functions/models/DurableOrchestrationClient.py @@ -546,3 +546,57 @@ def _get_raise_event_url( request_url += "?" + "&".join(query) return request_url + + async def rewind(self, + instance_id: str, + reason: str, + task_hub_name: Optional[str] = None, + connection_name: Optional[str] = None): + """Return / "rewind" a failed orchestration instance to a prior "healthy" state. + + Parameters + ---------- + instance_id: str + The ID of the orchestration instance to rewind. + reason: str + The reason for rewinding the orchestration instance. + task_hub_name: Optional[str] + The TaskHub of the orchestration to rewind + connection_name: Optional[str] + Name of the application setting containing the storage + connection string to use. + + Raises + ------ + Exception: + In case of a failure, it reports the reason for the exception + """ + request_url: str = "" + if self._orchestration_bindings.rpc_base_url: + path = f"instances/{instance_id}/rewind?reason={reason}" + query: List[str] = [] + if not (task_hub_name is None): + query.append(f"taskHub={task_hub_name}") + if not (connection_name is None): + query.append(f"connection={connection_name}") + if len(query) > 0: + path += "&" + "&".join(query) + + request_url = f"{self._orchestration_bindings.rpc_base_url}" + path + else: + raise Exception("The Python SDK only supports RPC endpoints." + + "Please remove the `localRpcEnabled` setting from host.json") + + response = await self._post_async_request(request_url, None) + status: int = response[0] + if status == 200 or status == 202: + return + elif status == 404: + ex_msg = f"No instance with ID {instance_id} found." + raise Exception(ex_msg) + elif status == 410: + ex_msg = "The rewind operation is only supported on failed orchestration instances." + raise Exception(ex_msg) + else: + ex_msg = response[1] + raise Exception(ex_msg) diff --git a/tests/models/test_DurableOrchestrationClient.py b/tests/models/test_DurableOrchestrationClient.py index 1b97629a..6a877568 100644 --- a/tests/models/test_DurableOrchestrationClient.py +++ b/tests/models/test_DurableOrchestrationClient.py @@ -19,6 +19,9 @@ MESSAGE_500 = 'instance failed with unhandled exception' MESSAGE_501 = "well we didn't expect that" +INSTANCE_ID = "2e2568e7-a906-43bd-8364-c81733c5891e" +REASON = "Stuff" + TEST_ORCHESTRATOR = "MyDurableOrchestrator" EXCEPTION_ORCHESTRATOR_NOT_FOUND_EXMESSAGE = "The function doesn't exist,"\ " is disabled, or is not an orchestrator function. Additional info: "\ @@ -540,3 +543,52 @@ async def test_start_new_orchestrator_internal_exception(binding_string): with pytest.raises(Exception) as ex: await client.start_new(TEST_ORCHESTRATOR) ex.match(status_str) + +@pytest.mark.asyncio +async def test_rewind_works_under_200_and_200_http_codes(binding_string): + """Tests that the rewind API works as expected under 'successful' http codes: 200, 202""" + client = DurableOrchestrationClient(binding_string) + for code in [200, 202]: + mock_request = MockRequest( + expected_url=f"{RPC_BASE_URL}instances/{INSTANCE_ID}/rewind?reason={REASON}", + response=[code, ""]) + client._post_async_request = mock_request.post + result = await client.rewind(INSTANCE_ID, REASON) + assert result is None + +@pytest.mark.asyncio +async def test_rewind_throws_exception_during_404_410_and_500_errors(binding_string): + """Tests the behaviour of rewind under 'exception' http codes: 404, 410, 500""" + client = DurableOrchestrationClient(binding_string) + codes = [404, 410, 500] + exception_strs = [ + f"No instance with ID {INSTANCE_ID} found.", + "The rewind operation is only supported on failed orchestration instances.", + "Something went wrong" + ] + for http_code, expected_exception_str in zip(codes, exception_strs): + mock_request = MockRequest( + expected_url=f"{RPC_BASE_URL}instances/{INSTANCE_ID}/rewind?reason={REASON}", + response=[http_code, "Something went wrong"]) + client._post_async_request = mock_request.post + + with pytest.raises(Exception) as ex: + await client.rewind(INSTANCE_ID, REASON) + ex_message = str(ex.value) + assert ex_message == expected_exception_str + +@pytest.mark.asyncio +async def test_rewind_with_no_rpc_endpoint(binding_string): + """Tests the behaviour of rewind without an RPC endpoint / under the legacy HTTP endpoint.""" + client = DurableOrchestrationClient(binding_string) + mock_request = MockRequest( + expected_url=f"{RPC_BASE_URL}instances/{INSTANCE_ID}/rewind?reason={REASON}", + response=[-1, ""]) + client._post_async_request = mock_request.post + client._orchestration_bindings._rpc_base_url = None + expected_exception_str = "The Python SDK only supports RPC endpoints."\ + + "Please remove the `localRpcEnabled` setting from host.json" + with pytest.raises(Exception) as ex: + await client.rewind(INSTANCE_ID, REASON) + ex_message = str(ex.value) + assert ex_message == expected_exception_str From 6c25a787af0dbdad287e91ce612dff1c2c8fae15 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 17 Sep 2020 10:41:20 -0700 Subject: [PATCH 13/28] Implemented out-of-proc error reporting schema (#196) --- azure/durable_functions/orchestrator.py | 11 +++- tests/orchestrator/test_call_http.py | 30 +++++---- tests/orchestrator/test_fan_out_fan_in.py | 29 ++++++--- tests/orchestrator/test_retries.py | 19 ++++-- .../test_sequential_orchestrator.py | 65 ++++++++++++++++--- ...test_sequential_orchestrator_with_retry.py | 28 +++++--- .../test_sub_orchestrator_with_retry.py | 30 +++++---- 7 files changed, 152 insertions(+), 60 deletions(-) diff --git a/azure/durable_functions/orchestrator.py b/azure/durable_functions/orchestrator.py index 9bb06fcf..70ee3fd8 100644 --- a/azure/durable_functions/orchestrator.py +++ b/azure/durable_functions/orchestrator.py @@ -100,13 +100,22 @@ def handle(self, context: DurableOrchestrationContext): actions=self.durable_context.actions, custom_status=self.durable_context.custom_status) except Exception as e: + exception_str = str(e) orchestration_state = OrchestratorState( is_done=False, output=None, # Should have no output, after generation range actions=self.durable_context.actions, - error=str(e), + error=exception_str, custom_status=self.durable_context.custom_status) + # Create formatted error, using out-of-proc error schema + error_label = "\n\n$OutOfProcData$:" + state_str = orchestration_state.to_json_string() + formatted_error = f"{exception_str}{error_label}{state_str}" + + # Raise exception, re-set stack to original location + raise Exception(formatted_error) from e + # No output if continue_as_new was called if self.durable_context.will_continue_as_new: orchestration_state._output = None diff --git a/tests/orchestrator/test_call_http.py b/tests/orchestrator/test_call_http.py index 53bcf539..cb0d95cb 100644 --- a/tests/orchestrator/test_call_http.py +++ b/tests/orchestrator/test_call_http.py @@ -104,17 +104,25 @@ def test_failed_state(): add_failed_http_events( context_builder, 0, failed_reason, failed_details) - result = get_orchestration_state_result( - context_builder, simple_get_generator_function) - - expected_state = base_expected_state() - request = get_request() - add_http_action(expected_state, request) - expected_state._error = f'{failed_reason} \n {failed_details}' - expected = expected_state.to_json() - - assert_valid_schema(result) - assert_orchestration_state_equals(expected, result) + try: + result = get_orchestration_state_result( + context_builder, simple_get_generator_function) + # We expected an exception + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + expected_state = base_expected_state() + request = get_request() + add_http_action(expected_state, request) + + error_msg = f'{failed_reason} \n {failed_details}' + expected_state._error = error_msg + state_str = expected_state.to_json_string() + + expected_error_str = f"{error_msg}{error_label}{state_str}" + assert expected_error_str == error_str def test_initial_post_state(): diff --git a/tests/orchestrator/test_fan_out_fan_in.py b/tests/orchestrator/test_fan_out_fan_in.py index 8a510460..5d0c33c5 100644 --- a/tests/orchestrator/test_fan_out_fan_in.py +++ b/tests/orchestrator/test_fan_out_fan_in.py @@ -153,13 +153,22 @@ def test_failed_parrot_value(): add_completed_task_set_events(context_builder, 1, 'ParrotValue', activity_count, 2, failed_reason, failed_details) - result = get_orchestration_state_result( - context_builder, generator_function) - - expected_state = base_expected_state(error=f'{failed_reason} \n {failed_details}') - add_single_action(expected_state, function_name='GetActivityCount', input_=None) - add_multi_actions(expected_state, function_name='ParrotValue', volume=activity_count) - expected = expected_state.to_json() - - assert_valid_schema(result) - assert_orchestration_state_equals(expected, result) + try: + result = get_orchestration_state_result( + context_builder, generator_function) + # we expected an exception + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + expected_state = base_expected_state(error=f'{failed_reason} \n {failed_details}') + add_single_action(expected_state, function_name='GetActivityCount', input_=None) + add_multi_actions(expected_state, function_name='ParrotValue', volume=activity_count) + + error_msg = f'{failed_reason} \n {failed_details}' + expected_state._error = error_msg + state_str = expected_state.to_json_string() + + expected_error_str = f"{error_msg}{error_label}{state_str}" + assert expected_error_str == error_str diff --git a/tests/orchestrator/test_retries.py b/tests/orchestrator/test_retries.py index 6e249c50..f1ca4fe7 100644 --- a/tests/orchestrator/test_retries.py +++ b/tests/orchestrator/test_retries.py @@ -255,9 +255,16 @@ def test_retries_can_fail(): """Tests the code path where a retry'ed Task fails""" context = get_context_with_retries(will_fail=True) - result = get_orchestration_state_result( - context, generator_function) - - expected_error = f"{REASONS} \n {DETAILS}" - assert "error" in result - assert result["error"] == expected_error \ No newline at end of file + try: + result = get_orchestration_state_result( + context, generator_function) + # We expected an exception + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + error_msg = f"{REASONS} \n {DETAILS}" + + expected_error_str = f"{error_msg}{error_label}" + assert str.startswith(error_str, expected_error_str) \ No newline at end of file diff --git a/tests/orchestrator/test_sequential_orchestrator.py b/tests/orchestrator/test_sequential_orchestrator.py index 731c0622..be031265 100644 --- a/tests/orchestrator/test_sequential_orchestrator.py +++ b/tests/orchestrator/test_sequential_orchestrator.py @@ -20,6 +20,18 @@ def generator_function(context): return outputs +def generator_function_rasing_ex(context): + outputs = [] + + task1 = yield context.call_activity("Hello", "Tokyo") + task2 = yield context.call_activity("Hello", "Seattle") + task3 = yield context.call_activity("Hello", "London") + + outputs.append(task1) + outputs.append(task2) + outputs.append(task3) + + raise ValueError("Oops!") def generator_function_with_serialization(context): """Ochestrator to test sequential activity calls with a serializable input arguments.""" @@ -99,17 +111,50 @@ def test_failed_tokyo_state(): add_hello_failed_events( context_builder, 0, failed_reason, failed_details) - result = get_orchestration_state_result( - context_builder, generator_function) - - expected_state = base_expected_state() - add_hello_action(expected_state, 'Tokyo') - expected_state._error = f'{failed_reason} \n {failed_details}' - expected = expected_state.to_json() - - assert_valid_schema(result) - assert_orchestration_state_equals(expected, result) + try: + result = get_orchestration_state_result( + context_builder, generator_function) + # expected an exception + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + expected_state = base_expected_state() + add_hello_action(expected_state, 'Tokyo') + error_msg = f'{failed_reason} \n {failed_details}' + expected_state._error = error_msg + state_str = expected_state.to_json_string() + + expected_error_str = f"{error_msg}{error_label}{state_str}" + assert expected_error_str == error_str + + +def test_user_code_raises_exception(): + context_builder = ContextBuilder('test_simple_function') + add_hello_completed_events(context_builder, 0, "\"Hello Tokyo!\"") + add_hello_completed_events(context_builder, 1, "\"Hello Seattle!\"") + add_hello_completed_events(context_builder, 2, "\"Hello London!\"") + try: + result = get_orchestration_state_result( + context_builder, generator_function_rasing_ex) + # expected an exception + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + expected_state = base_expected_state() + add_hello_action(expected_state, 'Tokyo') + add_hello_action(expected_state, 'Seattle') + add_hello_action(expected_state, 'London') + error_msg = 'Oops!' + expected_state._error = error_msg + state_str = expected_state.to_json_string() + + expected_error_str = f"{error_msg}{error_label}{state_str}" + assert expected_error_str == error_str def test_tokyo_and_seattle_state(): context_builder = ContextBuilder('test_simple_function') diff --git a/tests/orchestrator/test_sequential_orchestrator_with_retry.py b/tests/orchestrator/test_sequential_orchestrator_with_retry.py index aafd6ae7..0356b43b 100644 --- a/tests/orchestrator/test_sequential_orchestrator_with_retry.py +++ b/tests/orchestrator/test_sequential_orchestrator_with_retry.py @@ -198,13 +198,21 @@ def test_failed_tokyo_hit_max_attempts(): add_hello_failed_events(context_builder, 4, failed_reason, failed_details) add_retry_timer_events(context_builder, 5) - result = get_orchestration_state_result( - context_builder, generator_function) - - expected_state = base_expected_state() - add_hello_action(expected_state, 'Tokyo') - expected_state._error = f'{failed_reason} \n {failed_details}' - expected = expected_state.to_json() - - assert_valid_schema(result) - assert_orchestration_state_equals(expected, result) + try: + result = get_orchestration_state_result( + context_builder, generator_function) + # expected an exception + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + expected_state = base_expected_state() + add_hello_action(expected_state, 'Tokyo') + + error_msg = f'{failed_reason} \n {failed_details}' + expected_state._error = error_msg + state_str = expected_state.to_json_string() + + expected_error_str = f"{error_msg}{error_label}{state_str}" + assert expected_error_str == error_str diff --git a/tests/orchestrator/test_sub_orchestrator_with_retry.py b/tests/orchestrator/test_sub_orchestrator_with_retry.py index 3052ae6c..95b79811 100644 --- a/tests/orchestrator/test_sub_orchestrator_with_retry.py +++ b/tests/orchestrator/test_sub_orchestrator_with_retry.py @@ -109,15 +109,21 @@ def test_tokyo_and_seattle_and_london_state_all_failed(): add_hello_suborch_failed_events(context_builder, 4, failed_reason, failed_details) add_retry_timer_events(context_builder, 5) - - result = get_orchestration_state_result( - context_builder, generator_function) - - expected_state = base_expected_state() - add_hello_suborch_action(expected_state, 'Tokyo') - expected_state._error = f'{failed_reason} \n {failed_details}' - expected = expected_state.to_json() - expected_state._is_done = True - - #assert_valid_schema(result) - assert_orchestration_state_equals(expected, result) \ No newline at end of file + try: + result = get_orchestration_state_result( + context_builder, generator_function) + # Should have error'ed out + assert False + except Exception as e: + error_label = "\n\n$OutOfProcData$:" + error_str = str(e) + + expected_state = base_expected_state() + add_hello_suborch_action(expected_state, 'Tokyo') + + error_msg = f'{failed_reason} \n {failed_details}' + expected_state._error = error_msg + state_str = expected_state.to_json_string() + + expected_error_str = f"{error_msg}{error_label}{state_str}" + assert expected_error_str == error_str \ No newline at end of file From 188125748cfadc9dfcb6fa798b2111ca40450555 Mon Sep 17 00:00:00 2001 From: Wenonah Zhang Date: Tue, 6 Oct 2020 00:50:26 -0700 Subject: [PATCH 14/28] support signal entity --- .../models/DurableEntityContext.py | 4 +- .../models/DurableOrchestrationContext.py | 23 +++++++- .../models/actions/ActionType.py | 1 + .../models/actions/SignalEntityAction.py | 47 +++++++++++++++++ .../models/entities/EntityState.py | 2 +- azure/durable_functions/tasks/__init__.py | 2 + azure/durable_functions/tasks/call_entity.py | 9 +++- .../durable_functions/tasks/signal_entity.py | 52 +++++++++++++++++++ 8 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 azure/durable_functions/models/actions/SignalEntityAction.py create mode 100644 azure/durable_functions/tasks/signal_entity.py diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index c4c13879..ef91b13f 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -124,7 +124,9 @@ def set_state(self, state: Any) -> None: """ # TODO: enable serialization of custom types self._exists = True - self._state = json.dumps(state) + + # should only serialize the state at the end of a batch + self._state = state def get_state(self, initializer: Optional[Callable[[], Any]] = None) -> Any: """Get the current state of this entity. diff --git a/azure/durable_functions/models/DurableOrchestrationContext.py b/azure/durable_functions/models/DurableOrchestrationContext.py index f2d96186..de9300d4 100644 --- a/azure/durable_functions/models/DurableOrchestrationContext.py +++ b/azure/durable_functions/models/DurableOrchestrationContext.py @@ -12,7 +12,8 @@ from .utils.entity_utils import EntityId from ..tasks import call_activity_task, task_all, task_any, call_activity_with_retry_task, \ wait_for_external_event_task, continue_as_new, new_uuid, call_http, create_timer_task, \ - call_sub_orchestrator_task, call_sub_orchestrator_with_retry_task, call_entity_task + call_sub_orchestrator_task, call_sub_orchestrator_with_retry_task, call_entity_task, \ + signal_entity_task from azure.functions._durable_functions import _deserialize_custom_object @@ -380,6 +381,26 @@ def call_entity(self, entityId: EntityId, """ return call_entity_task(self.histories, entityId, operationName, operationInput) + def signal_entity(self, entityId: EntityId, + operationName: str, operationInput: Optional[Any] = None): + """Send a signal operation to Durable Entity given some input. + + Parameters + ---------- + entityId: EntityId + The ID of the entity to call + operationName: str + The operation to execute + operationInput: Optional[Any] + The input for tne operation, defaults to None. + + Returns + ------- + Task + A Task of the entity signal + """ + return signal_entity_task(self, self.histories, entityId, operationName, operationInput) + @property def will_continue_as_new(self) -> bool: """Return true if continue_as_new was called.""" diff --git a/azure/durable_functions/models/actions/ActionType.py b/azure/durable_functions/models/actions/ActionType.py index 66ffbb63..5b21af60 100644 --- a/azure/durable_functions/models/actions/ActionType.py +++ b/azure/durable_functions/models/actions/ActionType.py @@ -13,3 +13,4 @@ class ActionType(IntEnum): WAIT_FOR_EXTERNAL_EVENT: int = 6 CALL_ENTITY = 7 CALL_HTTP: int = 8 + SIGNAL_ENTITY: int = 9 \ No newline at end of file diff --git a/azure/durable_functions/models/actions/SignalEntityAction.py b/azure/durable_functions/models/actions/SignalEntityAction.py new file mode 100644 index 00000000..a2854254 --- /dev/null +++ b/azure/durable_functions/models/actions/SignalEntityAction.py @@ -0,0 +1,47 @@ +from typing import Any, Dict + +from .Action import Action +from .ActionType import ActionType +from ..utils.json_utils import add_attrib +from json import dumps +from azure.functions._durable_functions import _serialize_custom_object +from ..utils.entity_utils import EntityId + + +class SignalEntityAction(Action): + """Defines the structure of the Signal Entity object. + + Provides the information needed by the durable extension to be able to signal an entity + """ + + def __init__(self, entity_id: EntityId, operation: str, input_=None): + self.entity_id: EntityId = entity_id + + # Validating that EntityId exists before trying to parse its instanceId + if not self.entity_id: + raise ValueError("entity_id cannot be empty") + + self.instance_id: str = EntityId.get_scheduler_id(entity_id) + self.operation: str = operation + self.input_: str = dumps(input_, default=_serialize_custom_object) + + @property + def action_type(self) -> int: + """Get the type of action this class represents.""" + return ActionType.SIGNAL_ENTITY + + def to_json(self) -> Dict[str, Any]: + """Convert object into a json dictionary. + + Returns + ------- + Dict[str, Any] + The instance of the class converted into a json dictionary + """ + json_dict: Dict[str, Any] = {} + add_attrib(json_dict, self, "action_type", "actionType") + add_attrib(json_dict, self, 'instance_id', 'instanceId') + add_attrib(json_dict, self, 'operation', 'operation') + add_attrib(json_dict, self, 'input_', 'input') + + return json_dict \ No newline at end of file diff --git a/azure/durable_functions/models/entities/EntityState.py b/azure/durable_functions/models/entities/EntityState.py index 59fea5ad..e0de6b6e 100644 --- a/azure/durable_functions/models/entities/EntityState.py +++ b/azure/durable_functions/models/entities/EntityState.py @@ -55,7 +55,7 @@ def to_json(self) -> Dict[str, Any]: serialized_results = list(map(lambda x: x.to_json(), self.results)) json_dict["entityExists"] = self.entity_exists - json_dict["entityState"] = self.state + json_dict["entityState"] = json.dumps(self.state) json_dict["results"] = serialized_results json_dict["signals"] = self.signals return json_dict diff --git a/azure/durable_functions/tasks/__init__.py b/azure/durable_functions/tasks/__init__.py index 82eb8be0..5fec896b 100644 --- a/azure/durable_functions/tasks/__init__.py +++ b/azure/durable_functions/tasks/__init__.py @@ -12,6 +12,7 @@ from .call_http import call_http from .create_timer import create_timer_task from .call_entity import call_entity_task +from .signal_entity import signal_entity_task __all__ = [ 'call_activity_task', @@ -19,6 +20,7 @@ 'call_sub_orchestrator_task', 'call_sub_orchestrator_with_retry_task', 'call_entity_task', + 'signal_entity_task', 'call_http', 'continue_as_new', 'new_uuid', diff --git a/azure/durable_functions/tasks/call_entity.py b/azure/durable_functions/tasks/call_entity.py index 9d22989f..2e778553 100644 --- a/azure/durable_functions/tasks/call_entity.py +++ b/azure/durable_functions/tasks/call_entity.py @@ -8,7 +8,7 @@ from ..models.utils.entity_utils import EntityId from ..models.entities.RequestMessage import RequestMessage from ..models.entities.ResponseMessage import ResponseMessage - +import json def call_entity_task( state: List[HistoryEvent], @@ -64,7 +64,12 @@ def call_entity_task( if event_raised is not None: response = parse_history_event(event_raised) response = ResponseMessage.from_dict(response) - result = response.result + + # TODO: json.loads inside parse_history_event is not recursvie + # investigate if response.result is used elsewhere, + # which probably requires another deserialization + result = json.loads(response.result) + return Task( is_completed=True, is_faulted=False, diff --git a/azure/durable_functions/tasks/signal_entity.py b/azure/durable_functions/tasks/signal_entity.py new file mode 100644 index 00000000..b2045dcb --- /dev/null +++ b/azure/durable_functions/tasks/signal_entity.py @@ -0,0 +1,52 @@ +from typing import List, Any, Optional + +from ..models.Task import ( + Task) +from ..models.actions.SignalEntityAction import SignalEntityAction +from ..models.history import HistoryEvent, HistoryEventType +from .task_utilities import set_processed, parse_history_event, find_event +from ..models.utils.entity_utils import EntityId +from ..models.entities.RequestMessage import RequestMessage +from ..models.entities.ResponseMessage import ResponseMessage +import logging + + +def signal_entity_task( + context, + state: List[HistoryEvent], + entity_id: EntityId, + operation_name: str = "", + input_: Optional[Any] = None): + """ Signal a entity operation. + + It the action hasn't been scheduled, it appends the action. If the action has been scheduled, no ops. + + Parameters + ---------- + state: List[HistoryEvent] + The list of history events to search over to determine the + current state of the callEntity Task. + entity_id: EntityId + An identifier for the entity to call. + operation_name: str + The name of the operation the entity needs to execute. + input_: Any + The JSON-serializable input to pass to the activity function. + + """ + new_action = SignalEntityAction(entity_id, operation_name, input_) + scheduler_id = EntityId.get_scheduler_id(entity_id=entity_id) + + hist_type = HistoryEventType.EVENT_SENT + extra_constraints = { + "InstanceId": scheduler_id, + "Name": "op" + } + + event_sent = find_event(state, hist_type, extra_constraints) + set_processed([event_sent]) + + if event_sent: + return + + context.actions.append([new_action]) \ No newline at end of file From a30f63b40ee91c102e465cde3db427f1d1aa931c Mon Sep 17 00:00:00 2001 From: Wenonah Zhang Date: Tue, 6 Oct 2020 00:51:02 -0700 Subject: [PATCH 15/28] unit test and fixes in test infra --- .../models/test_DurableOrchestrationClient.py | 2 +- tests/orchestrator/test_entity.py | 36 ++++++++++++++++--- tests/test_utils/ContextBuilder.py | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/models/test_DurableOrchestrationClient.py b/tests/models/test_DurableOrchestrationClient.py index 1b97629a..188c52ea 100644 --- a/tests/models/test_DurableOrchestrationClient.py +++ b/tests/models/test_DurableOrchestrationClient.py @@ -492,7 +492,7 @@ async def test_wait_or_response_check_status_response(binding_string): @pytest.mark.asyncio -async def test_wait_or_response_check_status_response(binding_string): +async def test_wait_or_response_null_request(binding_string): status = dict(createdTime=TEST_CREATED_TIME, lastUpdatedTime=TEST_LAST_UPDATED_TIME, runtimeStatus="Running") diff --git a/tests/orchestrator/test_entity.py b/tests/orchestrator/test_entity.py index 9878e1ea..4d002d07 100644 --- a/tests/orchestrator/test_entity.py +++ b/tests/orchestrator/test_entity.py @@ -4,11 +4,13 @@ from azure.durable_functions.models.OrchestratorState import OrchestratorState from azure.durable_functions.models.actions.CallEntityAction \ import CallEntityAction +from azure.durable_functions.models.actions.SignalEntityAction \ + import SignalEntityAction from tests.test_utils.testClasses import SerializableClass import azure.durable_functions as df from typing import Any -def generator_function(context): +def generator_function_call_entity(context): outputs = [] entityId = df.EntityId("Counter", "myCounter") x = yield context.call_entity(entityId, "add", 3) @@ -16,15 +18,25 @@ def generator_function(context): outputs.append(x) return outputs +def generator_function_signal_entity(context): + outputs = [] + entityId = df.EntityId("Counter", "myCounter") + context.signal_entity(entityId, "add", 3) + x = yield context.call_entity(entityId, "get") + + outputs.append(x) + return outputs def base_expected_state(output=None) -> OrchestratorState: return OrchestratorState(is_done=False, actions=[], output=output) - def add_call_entity_action(state: OrchestratorState, id_: df.EntityId, op: str, input_: Any): action = CallEntityAction(entity_id=id_, operation=op, input_=input_) state.actions.append([action]) +def add_signal_entity_action(state: OrchestratorState, id_: df.EntityId, op: str, input_: Any): + action = SignalEntityAction(entity_id=id_, operation=op, input_=input_) + state.actions.append([action]) def add_call_entity_completed_events( context_builder: ContextBuilder, op: str, instance_id=str, input_=None): @@ -38,7 +50,7 @@ def test_call_entity_sent(): entityId = df.EntityId("Counter", "myCounter") result = get_orchestration_state_result( - context_builder, generator_function) + context_builder, generator_function_call_entity) expected_state = base_expected_state() add_call_entity_action(expected_state, entityId, "add", 3) @@ -46,11 +58,27 @@ def test_call_entity_sent(): #assert_valid_schema(result) assert_orchestration_state_equals(expected, result) + +def test_signal_entity_sent(): + context_builder = ContextBuilder('test_simple_function') + + entityId = df.EntityId("Counter", "myCounter") + result = get_orchestration_state_result( + context_builder, generator_function_signal_entity) + + expected_state = base_expected_state() + add_signal_entity_action(expected_state, entityId, "add", 3) + add_call_entity_action(expected_state, entityId, "get", None) + expected = expected_state.to_json() + + #assert_valid_schema(result) + assert_orchestration_state_equals(expected, result) + def test_call_entity_raised(): entityId = df.EntityId("Counter", "myCounter") context_builder = ContextBuilder('test_simple_function') - add_call_entity_completed_events(context_builder, "add", df.EntityId.get_scheduler_id(entityId),3) + add_call_entity_completed_events(context_builder, "add", df.EntityId.get_scheduler_id(entityId), 3) result = get_orchestration_state_result( context_builder, generator_function) diff --git a/tests/test_utils/ContextBuilder.py b/tests/test_utils/ContextBuilder.py index 8d7f646f..69b884cd 100644 --- a/tests/test_utils/ContextBuilder.py +++ b/tests/test_utils/ContextBuilder.py @@ -120,7 +120,7 @@ def add_event_raised_event(self, name:str, id_: int, input_=None, timestamp=None event = self.get_base_event(HistoryEventType.EVENT_RAISED, id_=id_, timestamp=timestamp) event.Name = name if is_entity: - event.Input = json.dumps({ "result": input_ }) + event.Input = json.dumps({ "result": json.dumps(input_) }) else: event.Input = input_ # event.timestamp = timestamp From 33fb7ca1d1392cb27cf87b15c45680d426579b11 Mon Sep 17 00:00:00 2001 From: Wenonah Zhang Date: Mon, 12 Oct 2020 14:13:18 -0700 Subject: [PATCH 16/28] lint --- .../durable_functions/models/DurableOrchestrationContext.py | 2 +- azure/durable_functions/models/actions/ActionType.py | 2 +- azure/durable_functions/models/actions/SignalEntityAction.py | 4 ++-- azure/durable_functions/tasks/__init__.py | 2 +- azure/durable_functions/tasks/call_entity.py | 5 +++-- azure/durable_functions/tasks/signal_entity.py | 4 ++-- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/azure/durable_functions/models/DurableOrchestrationContext.py b/azure/durable_functions/models/DurableOrchestrationContext.py index de9300d4..26dac506 100644 --- a/azure/durable_functions/models/DurableOrchestrationContext.py +++ b/azure/durable_functions/models/DurableOrchestrationContext.py @@ -382,7 +382,7 @@ def call_entity(self, entityId: EntityId, return call_entity_task(self.histories, entityId, operationName, operationInput) def signal_entity(self, entityId: EntityId, - operationName: str, operationInput: Optional[Any] = None): + operationName: str, operationInput: Optional[Any] = None): """Send a signal operation to Durable Entity given some input. Parameters diff --git a/azure/durable_functions/models/actions/ActionType.py b/azure/durable_functions/models/actions/ActionType.py index 5b21af60..406c6f86 100644 --- a/azure/durable_functions/models/actions/ActionType.py +++ b/azure/durable_functions/models/actions/ActionType.py @@ -13,4 +13,4 @@ class ActionType(IntEnum): WAIT_FOR_EXTERNAL_EVENT: int = 6 CALL_ENTITY = 7 CALL_HTTP: int = 8 - SIGNAL_ENTITY: int = 9 \ No newline at end of file + SIGNAL_ENTITY: int = 9 diff --git a/azure/durable_functions/models/actions/SignalEntityAction.py b/azure/durable_functions/models/actions/SignalEntityAction.py index a2854254..d6e9be54 100644 --- a/azure/durable_functions/models/actions/SignalEntityAction.py +++ b/azure/durable_functions/models/actions/SignalEntityAction.py @@ -43,5 +43,5 @@ def to_json(self) -> Dict[str, Any]: add_attrib(json_dict, self, 'instance_id', 'instanceId') add_attrib(json_dict, self, 'operation', 'operation') add_attrib(json_dict, self, 'input_', 'input') - - return json_dict \ No newline at end of file + + return json_dict diff --git a/azure/durable_functions/tasks/__init__.py b/azure/durable_functions/tasks/__init__.py index 5fec896b..9c7f6e9b 100644 --- a/azure/durable_functions/tasks/__init__.py +++ b/azure/durable_functions/tasks/__init__.py @@ -20,7 +20,7 @@ 'call_sub_orchestrator_task', 'call_sub_orchestrator_with_retry_task', 'call_entity_task', - 'signal_entity_task', + 'signal_entity_task', 'call_http', 'continue_as_new', 'new_uuid', diff --git a/azure/durable_functions/tasks/call_entity.py b/azure/durable_functions/tasks/call_entity.py index 2e778553..38ecb7cb 100644 --- a/azure/durable_functions/tasks/call_entity.py +++ b/azure/durable_functions/tasks/call_entity.py @@ -10,6 +10,7 @@ from ..models.entities.ResponseMessage import ResponseMessage import json + def call_entity_task( state: List[HistoryEvent], entity_id: EntityId, @@ -66,10 +67,10 @@ def call_entity_task( response = ResponseMessage.from_dict(response) # TODO: json.loads inside parse_history_event is not recursvie - # investigate if response.result is used elsewhere, + # investigate if response.result is used elsewhere, # which probably requires another deserialization result = json.loads(response.result) - + return Task( is_completed=True, is_faulted=False, diff --git a/azure/durable_functions/tasks/signal_entity.py b/azure/durable_functions/tasks/signal_entity.py index b2045dcb..7019c863 100644 --- a/azure/durable_functions/tasks/signal_entity.py +++ b/azure/durable_functions/tasks/signal_entity.py @@ -47,6 +47,6 @@ def signal_entity_task( set_processed([event_sent]) if event_sent: - return + return - context.actions.append([new_action]) \ No newline at end of file + context.actions.append([new_action]) From 7f350a2b7ea98c8e6a849c9acda904492bcfed47 Mon Sep 17 00:00:00 2001 From: Wenonah Zhang Date: Mon, 12 Oct 2020 14:19:50 -0700 Subject: [PATCH 17/28] lint and fix test --- azure/durable_functions/tasks/signal_entity.py | 14 ++++---------- tests/orchestrator/test_entity.py | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/azure/durable_functions/tasks/signal_entity.py b/azure/durable_functions/tasks/signal_entity.py index 7019c863..2f17d0ce 100644 --- a/azure/durable_functions/tasks/signal_entity.py +++ b/azure/durable_functions/tasks/signal_entity.py @@ -1,14 +1,8 @@ from typing import List, Any, Optional - -from ..models.Task import ( - Task) from ..models.actions.SignalEntityAction import SignalEntityAction from ..models.history import HistoryEvent, HistoryEventType -from .task_utilities import set_processed, parse_history_event, find_event +from .task_utilities import set_processed, find_event from ..models.utils.entity_utils import EntityId -from ..models.entities.RequestMessage import RequestMessage -from ..models.entities.ResponseMessage import ResponseMessage -import logging def signal_entity_task( @@ -17,9 +11,10 @@ def signal_entity_task( entity_id: EntityId, operation_name: str = "", input_: Optional[Any] = None): - """ Signal a entity operation. + """Signal a entity operation. - It the action hasn't been scheduled, it appends the action. If the action has been scheduled, no ops. + It the action hasn't been scheduled, it appends the action. + If the action has been scheduled, no ops. Parameters ---------- @@ -32,7 +27,6 @@ def signal_entity_task( The name of the operation the entity needs to execute. input_: Any The JSON-serializable input to pass to the activity function. - """ new_action = SignalEntityAction(entity_id, operation_name, input_) scheduler_id = EntityId.get_scheduler_id(entity_id=entity_id) diff --git a/tests/orchestrator/test_entity.py b/tests/orchestrator/test_entity.py index 4d002d07..ca10723b 100644 --- a/tests/orchestrator/test_entity.py +++ b/tests/orchestrator/test_entity.py @@ -81,7 +81,7 @@ def test_call_entity_raised(): add_call_entity_completed_events(context_builder, "add", df.EntityId.get_scheduler_id(entityId), 3) result = get_orchestration_state_result( - context_builder, generator_function) + context_builder, generator_function_call_entity) expected_state = base_expected_state( [3] From f047698242277f382641f101025d3f7e33a62914 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 19 Oct 2020 16:40:54 -0700 Subject: [PATCH 18/28] Added missing 'kind' field in TokenSource's serialized form (#209) --- azure/durable_functions/models/TokenSource.py | 2 ++ tests/models/test_TokenSource.py | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/models/test_TokenSource.py diff --git a/azure/durable_functions/models/TokenSource.py b/azure/durable_functions/models/TokenSource.py index d6ced05f..36b3c5f8 100644 --- a/azure/durable_functions/models/TokenSource.py +++ b/azure/durable_functions/models/TokenSource.py @@ -32,6 +32,7 @@ class ManagedIdentityTokenSource(TokenSource): def __init__(self, resource: str): super().__init__() self._resource: str = resource + self._kind: str = "AzureManagedIdentity" @property def resource(self) -> str: @@ -51,4 +52,5 @@ def to_json(self) -> Dict[str, Union[str, int]]: """ json_dict: Dict[str, Union[str, int]] = {} add_attrib(json_dict, self, 'resource') + json_dict["kind"] = self._kind return json_dict diff --git a/tests/models/test_TokenSource.py b/tests/models/test_TokenSource.py new file mode 100644 index 00000000..30dfe28f --- /dev/null +++ b/tests/models/test_TokenSource.py @@ -0,0 +1,11 @@ +from azure.durable_functions.models.TokenSource import ManagedIdentityTokenSource + +def test_serialization_fields(): + """Validates the TokenSource contains the expected fields when serialized to JSON""" + token_source = ManagedIdentityTokenSource(resource="TOKEN_SOURCE") + token_source_json = token_source.to_json() + + # Output JSON should contain a resource field and a kind field set to `AzureManagedIdentity` + assert "resource" in token_source_json.keys() + assert "kind" in token_source_json.keys() + assert token_source_json["kind"] == "AzureManagedIdentity" \ No newline at end of file From 59e77089d4d9934aff4c884b55e3df6f350d03fe Mon Sep 17 00:00:00 2001 From: David Justo Date: Tue, 20 Oct 2020 13:46:16 -0700 Subject: [PATCH 19/28] disabling schema validation (#212) --- tests/orchestrator/test_call_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/orchestrator/test_call_http.py b/tests/orchestrator/test_call_http.py index cb0d95cb..2eea39d6 100644 --- a/tests/orchestrator/test_call_http.py +++ b/tests/orchestrator/test_call_http.py @@ -136,7 +136,7 @@ def test_initial_post_state(): add_http_action(expected_state, request) expected = expected_state.to_json() - assert_valid_schema(result) + # assert_valid_schema(result) assert_orchestration_state_equals(expected, result) validate_result_http_request(result) @@ -170,6 +170,6 @@ def test_post_completed_state(): expected_state._is_done = True expected = expected_state.to_json() - assert_valid_schema(result) + # assert_valid_schema(result) assert_orchestration_state_equals(expected, result) validate_result_http_request(result) From dc9033062fc9440a94199798879a9f96a459df7f Mon Sep 17 00:00:00 2001 From: wenhzha <53274673+wenhzha@users.noreply.github.com> Date: Thu, 29 Oct 2020 09:56:05 -0700 Subject: [PATCH 20/28] Add Client Signal Entity (#216) * add client signal entity support * modify the formating --- .../models/DurableOrchestrationClient.py | 52 +++++++++++++++++++ .../models/RpcManagementOptions.py | 14 ++++- .../models/utils/entity_utils.py | 11 ++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/azure/durable_functions/models/DurableOrchestrationClient.py b/azure/durable_functions/models/DurableOrchestrationClient.py index 1442124a..390e0891 100644 --- a/azure/durable_functions/models/DurableOrchestrationClient.py +++ b/azure/durable_functions/models/DurableOrchestrationClient.py @@ -13,6 +13,7 @@ from .OrchestrationRuntimeStatus import OrchestrationRuntimeStatus from ..models.DurableOrchestrationBindings import DurableOrchestrationBindings from .utils.http_utils import get_async_request, post_async_request, delete_async_request +from .utils.entity_utils import EntityId from azure.functions._durable_functions import _serialize_custom_object @@ -457,6 +458,57 @@ async def wait_for_completion_or_create_check_status_response( else: return self.create_check_status_response(request, instance_id) + async def signal_entity(self, entityId: EntityId, operation_name: str, + operation_input: Optional[Any] = None, + task_hub_name: Optional[str] = None, + connection_name: Optional[str] = None) -> None: + """Signals an entity to perform an operation. + + Parameters + ---------- + entityId : EntityId + The EntityId of the targeted entity to perform operation. + operation_name: str + The name of the operation. + operation_input: Optional[Any] + The content for the operation. + task_hub_name: Optional[str] + The task hub name of the target entity. + connection_name: Optional[str] + The name of the connection string associated with [task_hub_name]. + + Raises + ------ + Exception: + When the signal entity call failed with an unexpected status code + + Returns + ------- + None + """ + options = RpcManagementOptions(operation_name=operation_name, + connection_name=connection_name, + task_hub_name=task_hub_name, + entity_Id=entityId) + + request_url = options.to_url(self._orchestration_bindings.rpc_base_url) + response = await self._post_async_request( + request_url, + json.dumps(operation_input) if operation_input else None) + + switch_statement = { + 202: lambda: None # signal accepted + } + + has_error_message = switch_statement.get( + response[0], + lambda: f"The operation failed with an unexpected status code {response[0]}") + + error_message = has_error_message() + + if error_message: + raise Exception(error_message) + @staticmethod def _create_http_response( status_code: int, body: Union[str, Any]) -> func.HttpResponse: diff --git a/azure/durable_functions/models/RpcManagementOptions.py b/azure/durable_functions/models/RpcManagementOptions.py index c16c508f..b41d1493 100644 --- a/azure/durable_functions/models/RpcManagementOptions.py +++ b/azure/durable_functions/models/RpcManagementOptions.py @@ -4,6 +4,8 @@ from azure.durable_functions.constants import DATETIME_STRING_FORMAT from azure.durable_functions.models.OrchestrationRuntimeStatus import OrchestrationRuntimeStatus +from .utils.entity_utils import EntityId + class RpcManagementOptions: """Class used to collect the options for getting orchestration status.""" @@ -12,7 +14,9 @@ def __init__(self, instance_id: str = None, task_hub_name: str = None, connection_name: str = None, show_history: bool = None, show_history_output: bool = None, created_time_from: datetime = None, created_time_to: datetime = None, - runtime_status: List[OrchestrationRuntimeStatus] = None, show_input: bool = None): + runtime_status: List[OrchestrationRuntimeStatus] = None, show_input: bool = None, + operation_name: str = None, + entity_Id: EntityId = None): self._instance_id = instance_id self._task_hub_name = task_hub_name self._connection_name = connection_name @@ -22,6 +26,8 @@ def __init__(self, instance_id: str = None, task_hub_name: str = None, self._created_time_to = created_time_to self._runtime_status = runtime_status self._show_input = show_input + self.operation_name = operation_name + self.entity_Id = entity_Id @staticmethod def _add_arg(query: List[str], name: str, value: Any): @@ -55,7 +61,10 @@ def to_url(self, base_url: Optional[str]) -> str: if base_url is None: raise ValueError("orchestration bindings has not RPC base url") - url = f"{base_url}instances/{self._instance_id if self._instance_id else ''}" + if self.entity_Id: + url = f'{base_url}{EntityId.get_entity_id_url_path(self.entity_Id)}' + else: + url = f"{base_url}instances/{self._instance_id if self._instance_id else ''}" query: List[str] = [] @@ -66,6 +75,7 @@ def to_url(self, base_url: Optional[str]) -> str: self._add_arg(query, 'showHistoryOutput', self._show_history_output) self._add_date_arg(query, 'createdTimeFrom', self._created_time_from) self._add_date_arg(query, 'createdTimeTo', self._created_time_to) + self._add_arg(query, 'op', self.operation_name) if self._runtime_status is not None and len(self._runtime_status) > 0: runtime_status = ",".join(r.value for r in self._runtime_status) self._add_arg(query, 'runtimeStatus', runtime_status) diff --git a/azure/durable_functions/models/utils/entity_utils.py b/azure/durable_functions/models/utils/entity_utils.py index 0ff33dd4..f5669323 100644 --- a/azure/durable_functions/models/utils/entity_utils.py +++ b/azure/durable_functions/models/utils/entity_utils.py @@ -69,6 +69,17 @@ def get_entity_id(scheduler_id: str) -> 'EntityId': [name, key] = components return EntityId(name, key) + @staticmethod + def get_entity_id_url_path(entity_id: 'EntityId') -> str: + """Print the the entity url path. + + Returns + ------- + str: + A url path of the EntityId + """ + return f'entities/{entity_id.name}/{entity_id.key}' + def __str__(self) -> str: """Print the string representation of this EntityId. From 0d9a32746c9ffdb1d55f124315c0cf7548538ab3 Mon Sep 17 00:00:00 2001 From: kemurayama <7068107+kemurayama@users.noreply.github.com> Date: Wed, 11 Nov 2020 18:54:23 +0900 Subject: [PATCH 21/28] Create link for sample folder It looked the brackets []() are hyper-link --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a08dbbd9..12863412 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,7 @@ Note: Conda based environments are not yet supported in Azure Functions. ### Setting up durable-py debugging -1. Git clone your fork and use any starter sample from this [folder] in your fork (https://github.com/Azure/azure-functions-durable-python/tree/dev/samples/) and open this folder in your VS Code editor. +1. Git clone your fork and use any starter sample from this [folder](https://github.com/Azure/azure-functions-durable-python/tree/dev/samples/) in your fork and open this folder in your VS Code editor. 2. Initialize this folder as an Azure Functions project using the VS Code Extension using these [instructions](https://docs.microsoft.com/en-us/azure/azure-functions/functions-create-first-function-vs-code?pivots=programming-language-python). This step will create a Python virtual environment if one doesn't exist already. From 4bb82dc56be82cb22ca89423d9b02c24c95d9c43 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 13 Nov 2020 10:30:16 -0800 Subject: [PATCH 22/28] added entitycontext, fixed sample --- samples/counter_entity/Counter/__init__.py | 2 +- tests/orchestrator/orchestrator_test_utils.py | 48 ++++++- tests/orchestrator/test_entity.py | 125 +++++++++++++++++- tests/test_utils/EntityContextBuilder.py | 57 ++++++++ 4 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 tests/test_utils/EntityContextBuilder.py diff --git a/samples/counter_entity/Counter/__init__.py b/samples/counter_entity/Counter/__init__.py index 821e1bd4..9c59d0f3 100644 --- a/samples/counter_entity/Counter/__init__.py +++ b/samples/counter_entity/Counter/__init__.py @@ -18,7 +18,7 @@ def entity_function(context: df.DurableEntityContext): for implementing durable entities. """ - current_value = context.get_state(lambda: "0") + current_value = context.get_state(lambda: 0) operation = context.operation_name if operation == "add": amount = context.get_input() diff --git a/tests/orchestrator/orchestrator_test_utils.py b/tests/orchestrator/orchestrator_test_utils.py index 9bdbb1b5..cef69472 100644 --- a/tests/orchestrator/orchestrator_test_utils.py +++ b/tests/orchestrator/orchestrator_test_utils.py @@ -1,19 +1,36 @@ import json -from typing import Callable, Iterator, Any, Dict +from typing import Callable, Iterator, Any, Dict, List from jsonschema import validate -from azure.durable_functions.models import DurableOrchestrationContext +from azure.durable_functions.models import DurableOrchestrationContext, DurableEntityContext from azure.durable_functions.orchestrator import Orchestrator +from azure.durable_functions.entity import Entity from .schemas.OrchetrationStateSchema import schema def assert_orchestration_state_equals(expected, result): + """Ensure that the observable OrchestratorState matches the expected result. + """ assert_attribute_equal(expected, result, "isDone") assert_actions_are_equal(expected, result) assert_attribute_equal(expected, result, "output") assert_attribute_equal(expected, result, "error") assert_attribute_equal(expected, result, "customStatus") +def assert_entity_state_equals(expected, result): + """Ensure the that the observable EntityState json matches the expected result. + """ + assert_attribute_equal(expected, result,"entityExists") + assert "results" in result + observed_results = result["results"] + expected_results = expected["results"] + assert_results_are_equal(expected_results, observed_results) + assert_attribute_equal(expected, result, "entityState") + assert_attribute_equal(expected, result, "signals") + +def assert_results_are_equal(expected: Dict[str, Any], result: Dict[str, Any]) -> bool: + assert_attribute_equal(expected, result, "result") + assert_attribute_equal(expected, result, "isError") def assert_attribute_equal(expected, result, attribute): if attribute in expected: @@ -50,6 +67,33 @@ def get_orchestration_state_result( result = json.loads(result_of_handle) return result +def get_entity_state_result( + context_builder: DurableEntityContext, + user_code: Callable[[DurableEntityContext], Any], + ) -> Dict[str, Any]: + """Simulate the result of running the entity function with the provided context and batch. + + Parameters + ---------- + context_builder: DurableEntityContext + A mocked entity context + user_code: Callable[[DurableEntityContext], Any] + A function implementing an entity + + Returns: + ------- + Dict[str, Any]: + JSON-response of the entity + """ + # The durable-extension automatically wraps the data within a 'self' key + context_as_string = context_builder.to_json_string() + entity = Entity(user_code) + + context, batch = DurableEntityContext.from_json(context_as_string) + result_of_handle = entity.handle(context, batch) + result = json.loads(result_of_handle) + return result + def get_orchestration_property( context_builder, activity_func: Callable[[DurableOrchestrationContext], Iterator[Any]], diff --git a/tests/orchestrator/test_entity.py b/tests/orchestrator/test_entity.py index ca10723b..eaf4bbc9 100644 --- a/tests/orchestrator/test_entity.py +++ b/tests/orchestrator/test_entity.py @@ -1,14 +1,18 @@ from .orchestrator_test_utils \ - import assert_orchestration_state_equals, get_orchestration_state_result, assert_valid_schema + import assert_orchestration_state_equals, get_orchestration_state_result, assert_valid_schema, \ + get_entity_state_result, assert_entity_state_equals from tests.test_utils.ContextBuilder import ContextBuilder +from tests.test_utils.EntityContextBuilder import EntityContextBuilder from azure.durable_functions.models.OrchestratorState import OrchestratorState +from azure.durable_functions.models.entities.EntityState import EntityState, OperationResult from azure.durable_functions.models.actions.CallEntityAction \ import CallEntityAction from azure.durable_functions.models.actions.SignalEntityAction \ import SignalEntityAction from tests.test_utils.testClasses import SerializableClass import azure.durable_functions as df -from typing import Any +from typing import Any, Dict, List +import json def generator_function_call_entity(context): outputs = [] @@ -27,6 +31,123 @@ def generator_function_signal_entity(context): outputs.append(x) return outputs +def counter_entity_function(context): + """A Counter Durable Entity. + + A simple example of a Durable Entity that implements + a simple counter. + """ + + current_value = context.get_state(lambda: 0) + operation = context.operation_name + if operation == "add": + amount = context.get_input() + current_value += amount + elif operation == "reset": + current_value = 0 + elif operation == "get": + pass + + result = f"The state is now: {current_value}" + context.set_state(current_value) + context.set_result(result) + + +def test_entity_signal_then_call(): + """Tests that a simple counter entity outputs the correct value + after a sequence of operations. Mostly just a sanity check. + """ + + # Create input batch + batch = [] + add_to_batch(batch, name="add", input_=3) + add_to_batch(batch, name="get") + context_builder = EntityContextBuilder(batch=batch) + + # Run the entity, get observed result + result = get_entity_state_result( + context_builder, + counter_entity_function, + ) + + # Construct expected result + expected_state = entity_base_expected_state() + apply_operation(expected_state, result="The state is now: 3", state=3) + expected = expected_state.to_json() + + # Ensure expectation matches observed behavior + #assert_valid_schema(result) + assert_entity_state_equals(expected, result) + + +def apply_operation(entity_state: EntityState, result: Any, state: Any, is_error: bool = False): + """Apply the effects of an operation to the expected entity state object + + Parameters + ---------- + entity_state: EntityState + The expected entity state object + result: Any + The result of the latest operation + state: Any + The state right after the latest operation + is_error: bool + Whether or not the operation resulted in an exception. Defaults to False + """ + entity_state.state = state + + # We cannot control duration, so default it to zero and avoid checking for it + # in later asserts + duration = 0 + operation_result = OperationResult( + is_error=is_error, + duration=duration, + result=result + ) + entity_state._results.append(operation_result) + +def add_to_batch(batch: List[Dict[str, Any]], name: str, input_: Any=None): + """Add new work item to the batch of entity operations. + + Parameters + ---------- + batch: List[Dict[str, Any]] + Current list of json-serialized entity work items + name: str + Name of the entity operation to be performed + input_: Optional[Any]: + Input to the operation. Defaults to None. + + Returns + -------- + List[Dict[str, str]]: + Batch of json-serialized entity work items + """ + # It is key to serialize the input twice, as this is + # the extension behavior + packet = { + "name": name, + "input": json.dumps(json.dumps(input_)) + } + batch.append(packet) + return batch + + +def entity_base_expected_state() -> EntityState: + """Get a base entity state. + + Returns + ------- + EntityState: + An EntityState with no results, no signals, a None state, and entity_exists set to True. + """ + return EntityState(results=[], signals=[], entity_exists=True, state=None) + +def add_call_entity_action_for_entity(state: OrchestratorState, id_: df.EntityId, op: str, input_: Any): + action = CallEntityAction(entity_id=id_, operation=op, input_=input_) + state.actions.append([action]) + + def base_expected_state(output=None) -> OrchestratorState: return OrchestratorState(is_done=False, actions=[], output=output) diff --git a/tests/test_utils/EntityContextBuilder.py b/tests/test_utils/EntityContextBuilder.py new file mode 100644 index 00000000..b2e26698 --- /dev/null +++ b/tests/test_utils/EntityContextBuilder.py @@ -0,0 +1,57 @@ +import json +from typing import Any, List, Dict, Any + +class EntityContextBuilder(): + """Mock class for an EntityContext object, includes a batch field for convenience + """ + def __init__(self, + name: str = "", + key: str = "", + exists: bool = True, + state: Any = None, + batch: List[Dict[str, Any]] = []): + """Construct an EntityContextBuilder + + Parameters + ---------- + name: str: + The name of the entity. Defaults to the empty string. + key: str + The key of the entity. Defaults to the empty string. + exists: bool + Boolean representing if the entity exists, defaults to True. + state: Any + The state of the entity, defaults ot None. + batch: List[Dict[str, Any]] + The upcoming batch of operations for the entity to perform. + Note that the batch is not technically a part of the entity context + and so it is here only for convenience. Defaults to the empty list. + """ + self.name = name + self.key = key + self.exists = exists + self.state = state + self.batch = batch + + def to_json_string(self) -> str: + """Generate a string-representation of the Entity input payload. + + The payload matches the current durable-extension entity-communication + schema. + + Returns + ------- + str: + A JSON-formatted string for an EntityContext to load via `from_json` + """ + context_json = { + "self": { + "name": self.name, + "key": self.key + }, + "state": self.state, + "exists": self.exists, + "batch": self.batch + } + json_string = json.dumps(context_json) + return json_string \ No newline at end of file From 5105a5455ba3c88f47b05d1d2353d76da0c0e482 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 13 Nov 2020 11:55:35 -0800 Subject: [PATCH 23/28] serialize custom types --- azure/durable_functions/entity.py | 10 +++++++--- .../models/DurableEntityContext.py | 14 ++++++-------- .../models/DurableOrchestrationClient.py | 1 - .../models/DurableOrchestrationContext.py | 1 - .../models/entities/EntityState.py | 3 ++- .../models/entities/OperationResult.py | 10 +++++----- .../durable_functions/models/entities/__init__.py | 2 +- azure/durable_functions/tasks/call_entity.py | 4 ++-- azure/durable_functions/tasks/task_utilities.py | 8 ++++---- samples/counter_entity/Counter/__init__.py | 1 - 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index 286361d0..eac0b065 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -47,7 +47,10 @@ def handle(self, context: DurableEntityContext, batch: List[Dict[str, Any]]) -> try: # populate context - context._operation = packet["name"] + operation = packet["name"] + if operation is None: + raise Exception("Entity operation is unassigned.") + context._operation = operation context._input = packet["input"] self.fn(context) result = context._result @@ -81,9 +84,10 @@ def create(cls, fn: Callable[[DurableEntityContext], None]) -> Callable[[Any], s Callable[[Any], str] Handle function of the newly created entity client """ - # TODO: review types here! def handle(context) -> str: - # TODO: this requires some commenting, where do we need to get this from the body + # It is not clear when the context JSON would be found + # inside a "body"-key, but this pattern matches the + # orchestrator implementation, so we keep it for safety. context_body = getattr(context, "body", None) if context_body is None: context_body = context diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index ef91b13f..1ece3fbe 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -71,8 +71,8 @@ def operation_name(self) -> Optional[str]: Optional[str] The current operation name """ - # TODO: Maybe we should raise an - # exception if _operation is None + if self._operation is None: + raise Exception("Entity operation is unassigned") return self._operation @property @@ -84,7 +84,7 @@ def is_newly_constructed(self) -> bool: bool True if the Entity was newly constructed. False otherwise. """ - # TODO: not updating this atm + # This is not updated at the moment, as its semantics are unclear return self._is_newly_constructed @classmethod @@ -122,10 +122,9 @@ def set_state(self, state: Any) -> None: state: Any The new state of the entity """ - # TODO: enable serialization of custom types self._exists = True - # should only serialize the state at the end of a batch + # should only serialize the state at the end of the batch self._state = state def get_state(self, initializer: Optional[Callable[[], Any]] = None) -> Any: @@ -142,11 +141,11 @@ def get_state(self, initializer: Optional[Callable[[], Any]] = None) -> Any: The current state of the entity """ state = self._state - # TODO: some weird errs here with None states if state is not None: return state elif initializer: - # TODO: ensure this is a fucntion + if not callable(initializer): + raise Exception("initializer argument needs to be a callable function") state = initializer() return state @@ -198,5 +197,4 @@ def from_json_util(self, json_str: str) -> Any: Any: The original datatype that was serialized """ - # TODO: this should be a util elsewhere, since we use it alot return json.loads(json_str, object_hook=_deserialize_custom_object) diff --git a/azure/durable_functions/models/DurableOrchestrationClient.py b/azure/durable_functions/models/DurableOrchestrationClient.py index 3b49634a..76554c5f 100644 --- a/azure/durable_functions/models/DurableOrchestrationClient.py +++ b/azure/durable_functions/models/DurableOrchestrationClient.py @@ -354,7 +354,6 @@ async def purge_instance_history_by( PurgeHistoryResult The results of the request to purge history """ - # TODO: do we really want folks to us this without specifying all the args? options = RpcManagementOptions(created_time_from=created_time_from, created_time_to=created_time_to, runtime_status=runtime_status) diff --git a/azure/durable_functions/models/DurableOrchestrationContext.py b/azure/durable_functions/models/DurableOrchestrationContext.py index 26dac506..4d632ad1 100644 --- a/azure/durable_functions/models/DurableOrchestrationContext.py +++ b/azure/durable_functions/models/DurableOrchestrationContext.py @@ -36,7 +36,6 @@ def __init__(self, self._new_uuid_counter: int = 0 self._sub_orchestrator_counter: int = 0 self._continue_as_new_flag: bool = False - # TODO: waiting on the `continue_as_new` intellisense until that's implemented self.decision_started_event: HistoryEvent = \ [e_ for e_ in self.histories if e_.event_type == HistoryEventType.ORCHESTRATOR_STARTED][0] diff --git a/azure/durable_functions/models/entities/EntityState.py b/azure/durable_functions/models/entities/EntityState.py index e0de6b6e..13d22e7e 100644 --- a/azure/durable_functions/models/entities/EntityState.py +++ b/azure/durable_functions/models/entities/EntityState.py @@ -1,5 +1,6 @@ from typing import List, Optional, Dict, Any from .Signal import Signal +from azure.functions._durable_functions import _serialize_custom_object from .OperationResult import OperationResult import json @@ -55,7 +56,7 @@ def to_json(self) -> Dict[str, Any]: serialized_results = list(map(lambda x: x.to_json(), self.results)) json_dict["entityExists"] = self.entity_exists - json_dict["entityState"] = json.dumps(self.state) + json_dict["entityState"] = json.dumps(self.state, default=_serialize_custom_object) json_dict["results"] = serialized_results json_dict["signals"] = self.signals return json_dict diff --git a/azure/durable_functions/models/entities/OperationResult.py b/azure/durable_functions/models/entities/OperationResult.py index b3c65499..de775a1e 100644 --- a/azure/durable_functions/models/entities/OperationResult.py +++ b/azure/durable_functions/models/entities/OperationResult.py @@ -1,4 +1,6 @@ from typing import Optional, Dict, Any +from azure.functions._durable_functions import _serialize_custom_object +import json class OperationResult: @@ -22,7 +24,6 @@ def __init__(self, result: Optional[str] The operation result. Defaults to None. """ - # TODO: perhaps this should inherit from orchestrator state self._is_error: bool = is_error self._duration: int = duration self._result: Optional[str] = result @@ -50,15 +51,14 @@ def duration(self) -> int: return self._duration @property - def result(self) -> Optional[str]: + def result(self) -> Any: """Get the operation's result. Returns ------- - Optional[str] + Any The operation's result """ - # TODO: is this necessary str or None? return self._result def to_json(self) -> Dict[str, Any]: @@ -72,5 +72,5 @@ def to_json(self) -> Dict[str, Any]: to_json: Dict[str, Any] = {} to_json["isError"] = self.is_error to_json["duration"] = self.duration - to_json["result"] = self.result + to_json["result"] = json.dumps(self.result, default=_serialize_custom_object) return to_json diff --git a/azure/durable_functions/models/entities/__init__.py b/azure/durable_functions/models/entities/__init__.py index 24eeda7f..6ecd233e 100644 --- a/azure/durable_functions/models/entities/__init__.py +++ b/azure/durable_functions/models/entities/__init__.py @@ -7,7 +7,7 @@ from .OperationResult import OperationResult from .EntityState import EntityState from .Signal import Signal -# TODO: this is very boilerplate-y, can we do better? + __all__ = [ 'RequestMessage', diff --git a/azure/durable_functions/tasks/call_entity.py b/azure/durable_functions/tasks/call_entity.py index 38ecb7cb..467e5b63 100644 --- a/azure/durable_functions/tasks/call_entity.py +++ b/azure/durable_functions/tasks/call_entity.py @@ -58,7 +58,7 @@ def call_entity_task( "Name": event_input.id } event_raised = find_event(state, hist_type, extra_constraints) - # TODO: does it make sense to have an event_sent but no `Input` attribute ?? + # TODO: does it make sense to have an event_sent but no `Input` attribute ? # If not, we should raise an exception here set_processed([event_sent, event_raised]) @@ -66,7 +66,7 @@ def call_entity_task( response = parse_history_event(event_raised) response = ResponseMessage.from_dict(response) - # TODO: json.loads inside parse_history_event is not recursvie + # TODO: json.loads inside parse_history_event is not recursive # investigate if response.result is used elsewhere, # which probably requires another deserialization result = json.loads(response.result) diff --git a/azure/durable_functions/tasks/task_utilities.py b/azure/durable_functions/tasks/task_utilities.py index 9cd1aadd..6da996c2 100644 --- a/azure/durable_functions/tasks/task_utilities.py +++ b/azure/durable_functions/tasks/task_utilities.py @@ -23,13 +23,12 @@ def parse_history_event(directive_result): # We provide the ability to deserialize custom objects, because the output of this # will be passed directly to the orchestrator as the output of some activity - # TODO: why do we have this chain of equivalent if-statements? if event_type == HistoryEventType.SUB_ORCHESTRATION_INSTANCE_COMPLETED: return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.TASK_COMPLETED: return json.loads(directive_result.Result, object_hook=_deserialize_custom_object) if event_type == HistoryEventType.EVENT_RAISED: - # TODO: why is the result here on input? + # TODO: Investigate why the payload is in "Input" instead of "Result" return json.loads(directive_result.Input, object_hook=_deserialize_custom_object) return None @@ -303,8 +302,9 @@ def gen_err_message(counter: int, mid_message: str, found: str, expected: str) - err = "Tried to lookup suborchestration in history but had not name to reference it." raise ValueError(err) - # TODO: The HistoryEvent does not necessarily have an name or an instance_id - # We should create sub-classes of these types like JS does + # TODO: The HistoryEvent does not necessarily have a name or an instance_id + # We should create sub-classes of these types like JS does, to ensure their + # precense. err_message: str = "" if not(event.Name == name): mid_message = "a function name of {} instead of the provided function name of {}." diff --git a/samples/counter_entity/Counter/__init__.py b/samples/counter_entity/Counter/__init__.py index 9c59d0f3..2caccd55 100644 --- a/samples/counter_entity/Counter/__init__.py +++ b/samples/counter_entity/Counter/__init__.py @@ -28,7 +28,6 @@ def entity_function(context: df.DurableEntityContext): elif operation == "get": pass - #TODO: we need to handle an invalid op, throw exception? context.set_state(current_value) context.set_result(current_value) From 4d6f22b531a3a1848a4d419589d594f5e20c5767 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 13 Nov 2020 12:30:35 -0800 Subject: [PATCH 24/28] sample uses signal --- azure/durable_functions/tasks/signal_entity.py | 3 +-- .../counter_entity/DurableOrchestration/__init__.py | 13 +++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/azure/durable_functions/tasks/signal_entity.py b/azure/durable_functions/tasks/signal_entity.py index 2f17d0ce..c7006495 100644 --- a/azure/durable_functions/tasks/signal_entity.py +++ b/azure/durable_functions/tasks/signal_entity.py @@ -39,8 +39,7 @@ def signal_entity_task( event_sent = find_event(state, hist_type, extra_constraints) set_processed([event_sent]) + context.actions.append([new_action]) if event_sent: return - - context.actions.append([new_action]) diff --git a/samples/counter_entity/DurableOrchestration/__init__.py b/samples/counter_entity/DurableOrchestration/__init__.py index 11ddfe5b..33bc7e6b 100644 --- a/samples/counter_entity/DurableOrchestration/__init__.py +++ b/samples/counter_entity/DurableOrchestration/__init__.py @@ -15,7 +15,7 @@ def orchestrator_function(context: df.DurableOrchestrationContext): """This function provides the a simple implementation of an orchestrator - that calls a counter Durable Entity. + that signals and then calls a counter Durable Entity. Parameters ---------- @@ -26,17 +26,10 @@ def orchestrator_function(context: df.DurableOrchestrationContext): ------- state The state after applying the operation on the Durable Entity - - Yields - ------- - call_entity: Any - Yields to wait on the Durable Entity to process its operation, - with the provided input, to then receive its counter state value. """ - - # TODO: make the operation and the input into URL params entityId = df.EntityId("Counter", "myCounter") - state = yield context.call_entity(entityId, "add", 3) + context.signal_entity(entityId, "add", 3) + state = yield context.call_entity(entityId, "get") return state main = df.Orchestrator.create(orchestrator_function) \ No newline at end of file From af482d2fc1722ada9d639af46b918999def9fe95 Mon Sep 17 00:00:00 2001 From: Carl Vitzthum Date: Wed, 18 Nov 2020 18:39:20 -0500 Subject: [PATCH 25/28] Serialize input for CallActivityWithRetryAction (#225) Co-authored-by: cvitzthum-nasuni --- .../actions/CallActivityWithRetryAction.py | 4 +- tests/orchestrator/test_retries.py | 49 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/azure/durable_functions/models/actions/CallActivityWithRetryAction.py b/azure/durable_functions/models/actions/CallActivityWithRetryAction.py index 7ec97580..a6b33288 100644 --- a/azure/durable_functions/models/actions/CallActivityWithRetryAction.py +++ b/azure/durable_functions/models/actions/CallActivityWithRetryAction.py @@ -1,9 +1,11 @@ +from json import dumps from typing import Dict, Union from .Action import Action from .ActionType import ActionType from ..RetryOptions import RetryOptions from ..utils.json_utils import add_attrib, add_json_attrib +from azure.functions._durable_functions import _serialize_custom_object class CallActivityWithRetryAction(Action): @@ -16,7 +18,7 @@ def __init__(self, function_name: str, retry_options: RetryOptions, input_=None): self.function_name: str = function_name self.retry_options: RetryOptions = retry_options - self.input_ = input_ + self.input_ = dumps(input_, default=_serialize_custom_object) if not self.function_name: raise ValueError("function_name cannot be empty") diff --git a/tests/orchestrator/test_retries.py b/tests/orchestrator/test_retries.py index f1ca4fe7..c08ffdad 100644 --- a/tests/orchestrator/test_retries.py +++ b/tests/orchestrator/test_retries.py @@ -1,4 +1,5 @@ from tests.test_utils.ContextBuilder import ContextBuilder +from tests.test_utils.testClasses import SerializableClass from azure.durable_functions.models.RetryOptions import RetryOptions from azure.durable_functions.models.OrchestratorState import OrchestratorState from azure.durable_functions.models.DurableOrchestrationContext import DurableOrchestrationContext @@ -42,6 +43,38 @@ def generator_function(context: DurableOrchestrationContext): return outputs + +def generator_function_with_serialization(context: DurableOrchestrationContext): + """Orchestrator function for testing retry'ing with serializable input arguments. + + Parameters + ---------- + context: DurableOrchestrationContext + Durable orchestration context, exposes the Durable API + + Returns + ------- + List[str]: + Output of activities, a list of hello'd cities + """ + + outputs = [] + + retry_options = RETRY_OPTIONS + task1 = yield context.call_activity_with_retry( + "Hello", retry_options, SerializableClass("Tokyo")) + task2 = yield context.call_activity_with_retry( + "Hello", retry_options, SerializableClass("Seatlle")) + task3 = yield context.call_activity_with_retry( + "Hello", retry_options, SerializableClass("London")) + + outputs.append(task1) + outputs.append(task2) + outputs.append(task3) + + return outputs + + def get_context_with_retries_and_corrupted_completion() -> ContextBuilder: """Get a ContextBuilder whose history contains a late completion event for an event that already failed. @@ -267,4 +300,18 @@ def test_retries_can_fail(): error_msg = f"{REASONS} \n {DETAILS}" expected_error_str = f"{error_msg}{error_label}" - assert str.startswith(error_str, expected_error_str) \ No newline at end of file + assert str.startswith(error_str, expected_error_str) + +def test_retries_with_serializable_input(): + """Tests that retried tasks work with serialized input classes.""" + context = get_context_with_retries() + + result_1 = get_orchestration_state_result( + context, generator_function) + + result_2 = get_orchestration_state_result( + context, generator_function_with_serialization) + + assert "output" in result_1 + assert "output" in result_2 + assert result_1["output"] == result_2["output"] From d95c56dae289a3b0143c229e3c835b47b49cdbf4 Mon Sep 17 00:00:00 2001 From: kemurayama <7068107+kemurayama@users.noreply.github.com> Date: Thu, 3 Dec 2020 02:00:34 +0900 Subject: [PATCH 26/28] Replace datetime.strptime for find_task_timer_created() (#228) * Replace datetime.strptime * Deleted unnecessary format string * Delete whitespace --- azure/durable_functions/tasks/task_utilities.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure/durable_functions/tasks/task_utilities.py b/azure/durable_functions/tasks/task_utilities.py index 3c54d776..aa7e8f28 100644 --- a/azure/durable_functions/tasks/task_utilities.py +++ b/azure/durable_functions/tasks/task_utilities.py @@ -1,8 +1,7 @@ import json from ..models.history import HistoryEventType, HistoryEvent -from ..constants import DATETIME_STRING_FORMAT from azure.functions._durable_functions import _deserialize_custom_object -from datetime import datetime +from dateutil import parser from typing import List, Optional from ..models.actions.Action import Action from ..models.Task import Task @@ -140,7 +139,7 @@ def find_task_timer_created(state, fire_at): tasks = [] for e in state: if e.event_type == HistoryEventType.TIMER_CREATED and hasattr(e, "FireAt"): - if datetime.strptime(e.FireAt, DATETIME_STRING_FORMAT) == fire_at: + if parser.parse(e.FireAt).replace(tzinfo=None) == fire_at: tasks.append(e) if len(tasks) == 0: From 85192e59142d1a9946c64711a34fcf8f7e8ca5f8 Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 2 Dec 2020 22:57:07 -0800 Subject: [PATCH 27/28] PR feedback --- azure/durable_functions/entity.py | 10 ++--- .../models/DurableEntityContext.py | 38 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index eac0b065..dd026c3f 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -7,7 +7,7 @@ class Entity: """Durable Entity Class. - Responsible for execuitng the user-defined entity function. + Responsible for executing the user-defined entity function. """ def __init__(self, entity_func: Callable[[DurableEntityContext], None]): @@ -40,18 +40,18 @@ def handle(self, context: DurableEntityContext, batch: List[Dict[str, Any]]) -> entity execution. """ response = EntityState(results=[], signals=[]) - for packet in batch: + for operation_data in batch: result: Any = None is_error: bool = False start_time: datetime = datetime.now() try: # populate context - operation = packet["name"] + operation = operation_data["name"] if operation is None: - raise Exception("Entity operation is unassigned.") + raise Exception("Durable Functions Internal Error: Entity operation was missing a name field") context._operation = operation - context._input = packet["input"] + context._input = operation_data["input"] self.fn(context) result = context._result diff --git a/azure/durable_functions/models/DurableEntityContext.py b/azure/durable_functions/models/DurableEntityContext.py index 1ece3fbe..cc1d9814 100644 --- a/azure/durable_functions/models/DurableEntityContext.py +++ b/azure/durable_functions/models/DurableEntityContext.py @@ -109,8 +109,8 @@ def from_json(cls, json_str: str) -> Tuple['DurableEntityContext', List[Dict[str serialized_state = json_dict["state"] if serialized_state is not None: - json_dict["state"] = json.loads(serialized_state, - object_hook=_deserialize_custom_object) + json_dict["state"] = from_json_util(serialized_state) + batch = json_dict.pop("batch") return cls(**json_dict), batch @@ -160,7 +160,7 @@ def get_input(self) -> Any: input_ = None req_input = self._input req_input = json.loads(req_input) - input_ = None if req_input is None else self.from_json_util(req_input) + input_ = None if req_input is None else from_json_util(req_input) return input_ def set_result(self, result: Any) -> None: @@ -179,22 +179,22 @@ def destruct_on_exit(self) -> None: self._exists = False self._state = None - def from_json_util(self, json_str: str) -> Any: - """Load an arbitrary datatype from its JSON representation. +def from_json_util(self, json_str: str) -> Any: + """Load an arbitrary datatype from its JSON representation. - The Out-of-proc SDK has a special JSON encoding strategy - to enable arbitrary datatypes to be serialized. This utility - loads a JSON with the assumption that it follows that encoding - method. + The Out-of-proc SDK has a special JSON encoding strategy + to enable arbitrary datatypes to be serialized. This utility + loads a JSON with the assumption that it follows that encoding + method. - Parameters - ---------- - json_str: str - A JSON-formatted string, from durable-extension + Parameters + ---------- + json_str: str + A JSON-formatted string, from durable-extension - Returns - ------- - Any: - The original datatype that was serialized - """ - return json.loads(json_str, object_hook=_deserialize_custom_object) + Returns + ------- + Any: + The original datatype that was serialized + """ + return json.loads(json_str, object_hook=_deserialize_custom_object) From 274d4c00477786017efc88f6c1b8881a9bd5337d Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 3 Dec 2020 11:13:46 -0800 Subject: [PATCH 28/28] not handling internal exceptions, deferring to extension --- azure/durable_functions/entity.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure/durable_functions/entity.py b/azure/durable_functions/entity.py index dd026c3f..3c278ff6 100644 --- a/azure/durable_functions/entity.py +++ b/azure/durable_functions/entity.py @@ -3,6 +3,8 @@ from datetime import datetime from typing import Callable, Any, List, Dict +class InternalEntityException(Exception): + pass class Entity: """Durable Entity Class. @@ -49,12 +51,15 @@ def handle(self, context: DurableEntityContext, batch: List[Dict[str, Any]]) -> # populate context operation = operation_data["name"] if operation is None: - raise Exception("Durable Functions Internal Error: Entity operation was missing a name field") + raise InternalEntityException("Durable Functions Internal Error: Entity operation was missing a name field") context._operation = operation context._input = operation_data["input"] self.fn(context) result = context._result + except InternalEntityException as e: + raise e + except Exception as e: is_error = True result = str(e)