From 90df3594f1fdf8a40b3d4d5eab702e7811e86e6a Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 23 May 2022 16:11:50 -0400 Subject: [PATCH 01/37] feat: WIP profiling support for Python --- ddtrace/profiling/exporter/http.py | 1 + ddtrace/profiling/recorder.py | 1 + setup.py | 9 ++++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index d3405200a63..fe0c0c48e6b 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -77,6 +77,7 @@ def __attrs_post_init__(self): tags.update({k: six.ensure_binary(v) for k, v in self.tags.items()}) tags.update( { + #"function_name": os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode(), "host": HOSTNAME.encode("utf-8"), "language": b"python", "runtime": PYTHON_IMPLEMENTATION, diff --git a/ddtrace/profiling/recorder.py b/ddtrace/profiling/recorder.py index 3bfc5cb46f0..03351cabeb2 100644 --- a/ddtrace/profiling/recorder.py +++ b/ddtrace/profiling/recorder.py @@ -77,6 +77,7 @@ def push_events(self, events): q.extend(events) def _get_deque_for_event_type(self, event_type): + print(f'MAX EVENTS: {self.max_events}, EVENT TYPE: {event_type}') return collections.deque(maxlen=self.max_events.get(event_type, self.default_max_events)) def _reset_events(self): diff --git a/setup.py b/setup.py index eeaef56f4af..3f6078f952a 100644 --- a/setup.py +++ b/setup.py @@ -210,7 +210,7 @@ def get_exts_for(name): "bytecode; python_version>='3.8'", ] - +extra_compile_args.append('-std=gnu99') setup( name="ddtrace", description="Datadog APM client library", @@ -296,11 +296,13 @@ def get_exts_for(name): "ddtrace.internal._rand", sources=["ddtrace/internal/_rand.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.internal._tagset", sources=["ddtrace/internal/_tagset.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Extension( "ddtrace.internal._encoding", @@ -319,26 +321,31 @@ def get_exts_for(name): "ddtrace.profiling.collector._traceback", sources=["ddtrace/profiling/collector/_traceback.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling._threading", sources=["ddtrace/profiling/_threading.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling.collector._task", sources=["ddtrace/profiling/collector/_task.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling.exporter.pprof", sources=["ddtrace/profiling/exporter/pprof.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling._build", sources=["ddtrace/profiling/_build.pyx"], language="c", + extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.appsec._ddwaf", From 0f3fd528dfc3b39e8db85afe18efc751de742a98 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 31 May 2022 12:28:38 -0400 Subject: [PATCH 02/37] wip: function name as tag --- ddtrace/profiling/exporter/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index fe0c0c48e6b..c5225a11223 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -77,7 +77,7 @@ def __attrs_post_init__(self): tags.update({k: six.ensure_binary(v) for k, v in self.tags.items()}) tags.update( { - #"function_name": os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode(), + "function_name": os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode(), "host": HOSTNAME.encode("utf-8"), "language": b"python", "runtime": PYTHON_IMPLEMENTATION, From c06cd4f9d0144c97dbf24740410947e671a16858 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 1 Jun 2022 15:01:11 -0400 Subject: [PATCH 03/37] feat: custom serverless scheduler to wake up every second and check if we've acquired 60s of data before flushing --- ddtrace/profiling/profiler.py | 4 +- ddtrace/profiling/recorder.py | 14 ++++- ddtrace/profiling/serverless_scheduler.py | 69 +++++++++++++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 ddtrace/profiling/serverless_scheduler.py diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 9888ea31f1f..01b83eb2af9 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -18,7 +18,7 @@ from ddtrace.profiling import collector from ddtrace.profiling import exporter from ddtrace.profiling import recorder -from ddtrace.profiling import scheduler +from ddtrace.profiling import serverless_scheduler from ddtrace.profiling.collector import asyncio from ddtrace.profiling.collector import memalloc from ddtrace.profiling.collector import stack @@ -209,7 +209,7 @@ def __attrs_post_init__(self): exporters = self._build_default_exporters() if exporters: - self._scheduler = scheduler.Scheduler( + self._scheduler = serverless_scheduler.ServerlessScheduler( recorder=r, exporters=exporters, before_flush=self._collectors_snapshot ) diff --git a/ddtrace/profiling/recorder.py b/ddtrace/profiling/recorder.py index 03351cabeb2..3d5e07daea2 100644 --- a/ddtrace/profiling/recorder.py +++ b/ddtrace/profiling/recorder.py @@ -77,7 +77,6 @@ def push_events(self, events): q.extend(events) def _get_deque_for_event_type(self, event_type): - print(f'MAX EVENTS: {self.max_events}, EVENT TYPE: {event_type}') return collections.deque(maxlen=self.max_events.get(event_type, self.default_max_events)) def _reset_events(self): @@ -95,3 +94,16 @@ def reset(self): events = self.events self._reset_events() return events + + def events_delta(self): + oldest, newest = None, None + for k, v in self.events.items(): + if oldest: + oldest = v[0].timestamp if v[0].timestamp < oldest else oldest + else: + oldest = v[0].timestamp + if newest: + newest = v[-1].timestamp if v[0].timestamp < newest else newest + else: + newest = v[-1].timestamp + return newest - oldest diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py new file mode 100644 index 00000000000..65a15d6296e --- /dev/null +++ b/ddtrace/profiling/serverless_scheduler.py @@ -0,0 +1,69 @@ +# -*- encoding: utf-8 -*- +import logging + +import attr + +from ddtrace.internal import compat +from ddtrace.internal import periodic +from ddtrace.internal.utils import attr as attr_utils +from ddtrace.profiling import _traceback +from ddtrace.profiling import exporter + + +LOG = logging.getLogger(__name__) + + +@attr.s +class ServerlessScheduler(periodic.PeriodicService): + """Schedule export of recorded data for FaaS systems.""" + """Collects profiles until a total of 60s of data, then flushes""" + + recorder = attr.ib() + exporters = attr.ib() + before_flush = attr.ib(default=None, eq=False) + _interval = attr.ib(factory=attr_utils.from_env("DD_SERVERLESS_PROFILING_UPLOAD_INTERVAL", 1.0, float)) + _configured_interval = attr.ib(init=False) + _last_export = attr.ib(init=False, default=None, eq=False) + + def __attrs_post_init__(self): + # Copy the value to use it later since we're going to adjust the real interval + self._configured_interval = self.interval + + def _start_service(self): # type: ignore[override] + # type: (...) -> None + """Start the scheduler.""" + LOG.debug("Starting scheduler") + super(ServerlessScheduler, self)._start_service() + self._last_export = compat.time_ns() + LOG.debug("Scheduler started") + + def flush(self): + """Flush events from recorder to exporters.""" + LOG.debug("Flushing events") + if self.before_flush is not None: + try: + self.before_flush() + except Exception: + LOG.error("Scheduler before_flush hook failed", exc_info=True) + if self.exporters: + events = self.recorder.reset() + start = self._last_export + self._last_export = compat.time_ns() + for exp in self.exporters: + try: + exp.export(events, start, self._last_export) + except exporter.ExportError as e: + LOG.error("Unable to export profile: %s. Ignoring.", _traceback.format_exception(e)) + except Exception: + LOG.exception( + "Unexpected error while exporting events. " + "Please report this bug to https://github.com/DataDog/dd-trace-py/issues" + ) + + def periodic(self): + if self.recorder.events_delta() > 60.0: + start_time = compat.monotonic() + try: + self.flush() + finally: + self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) From 5c9ec7bc8aea295cc56174fca542eb397c806b75 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 10 Jun 2022 12:40:16 -0400 Subject: [PATCH 04/37] feat: Conditionally use serverless scheduler. Refactor to inherit from base scheduler and override periodic method --- ddtrace/profiling/exporter/http.py | 6 ++- ddtrace/profiling/profiler.py | 13 +++-- ddtrace/profiling/recorder.py | 15 +----- ddtrace/profiling/serverless_scheduler.py | 58 ++++------------------- 4 files changed, 24 insertions(+), 68 deletions(-) diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index c5225a11223..5dd11bb5f16 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -77,7 +77,6 @@ def __attrs_post_init__(self): tags.update({k: six.ensure_binary(v) for k, v in self.tags.items()}) tags.update( { - "function_name": os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode(), "host": HOSTNAME.encode("utf-8"), "language": b"python", "runtime": PYTHON_IMPLEMENTATION, @@ -85,6 +84,11 @@ def __attrs_post_init__(self): "profiler_version": ddtrace.__version__.encode("ascii"), } ) + lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode() + if lambda_function_name: + tags.update({ + "functionname": lambda_function_name + }) if self.version: tags["version"] = self.version.encode("utf-8") diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 01b83eb2af9..f6f31a19344 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -19,6 +19,7 @@ from ddtrace.profiling import exporter from ddtrace.profiling import recorder from ddtrace.profiling import serverless_scheduler +from ddtrace.profiling import scheduler from ddtrace.profiling.collector import asyncio from ddtrace.profiling.collector import memalloc from ddtrace.profiling.collector import stack @@ -209,9 +210,15 @@ def __attrs_post_init__(self): exporters = self._build_default_exporters() if exporters: - self._scheduler = serverless_scheduler.ServerlessScheduler( + lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode() + if lambda_function_name: + self._scheduler = serverless_scheduler.ServerlessScheduler( recorder=r, exporters=exporters, before_flush=self._collectors_snapshot - ) + ) + else: + self._scheduler = scheduler.Scheduler( + recorder=r, exporters=exporters, before_flush=self._collectors_snapshot + ) self.set_asyncio_event_loop_policy() @@ -283,4 +290,4 @@ def _stop_service( # type: ignore[override] pass for col in reversed(self._collectors): - col.join() + col.join() \ No newline at end of file diff --git a/ddtrace/profiling/recorder.py b/ddtrace/profiling/recorder.py index 3d5e07daea2..7e68b1b9413 100644 --- a/ddtrace/profiling/recorder.py +++ b/ddtrace/profiling/recorder.py @@ -93,17 +93,4 @@ def reset(self): with self._events_lock: events = self.events self._reset_events() - return events - - def events_delta(self): - oldest, newest = None, None - for k, v in self.events.items(): - if oldest: - oldest = v[0].timestamp if v[0].timestamp < oldest else oldest - else: - oldest = v[0].timestamp - if newest: - newest = v[-1].timestamp if v[0].timestamp < newest else newest - else: - newest = v[-1].timestamp - return newest - oldest + return events \ No newline at end of file diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 65a15d6296e..21a3a2d518a 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -2,68 +2,26 @@ import logging import attr - from ddtrace.internal import compat -from ddtrace.internal import periodic +from ddtrace.profiling import scheduler from ddtrace.internal.utils import attr as attr_utils -from ddtrace.profiling import _traceback -from ddtrace.profiling import exporter LOG = logging.getLogger(__name__) @attr.s -class ServerlessScheduler(periodic.PeriodicService): - """Schedule export of recorded data for FaaS systems.""" - """Collects profiles until a total of 60s of data, then flushes""" - - recorder = attr.ib() - exporters = attr.ib() - before_flush = attr.ib(default=None, eq=False) - _interval = attr.ib(factory=attr_utils.from_env("DD_SERVERLESS_PROFILING_UPLOAD_INTERVAL", 1.0, float)) - _configured_interval = attr.ib(init=False) - _last_export = attr.ib(init=False, default=None, eq=False) - - def __attrs_post_init__(self): - # Copy the value to use it later since we're going to adjust the real interval - self._configured_interval = self.interval - - def _start_service(self): # type: ignore[override] - # type: (...) -> None - """Start the scheduler.""" - LOG.debug("Starting scheduler") - super(ServerlessScheduler, self)._start_service() - self._last_export = compat.time_ns() - LOG.debug("Scheduler started") - - def flush(self): - """Flush events from recorder to exporters.""" - LOG.debug("Flushing events") - if self.before_flush is not None: - try: - self.before_flush() - except Exception: - LOG.error("Scheduler before_flush hook failed", exc_info=True) - if self.exporters: - events = self.recorder.reset() - start = self._last_export - self._last_export = compat.time_ns() - for exp in self.exporters: - try: - exp.export(events, start, self._last_export) - except exporter.ExportError as e: - LOG.error("Unable to export profile: %s. Ignoring.", _traceback.format_exception(e)) - except Exception: - LOG.exception( - "Unexpected error while exporting events. " - "Please report this bug to https://github.com/DataDog/dd-trace-py/issues" - ) +class ServerlessScheduler(scheduler.Scheduler): + _interval = attr.ib(factory=attr_utils.from_env("DD_SERVERLESS_PROFILING_UPLOAD_INTERVAL", 1, float)) + _total_profiled_seconds = attr.ib(default=0) def periodic(self): - if self.recorder.events_delta() > 60.0: + if self._total_profiled_seconds > self._interval: start_time = compat.monotonic() try: self.flush() + self._total_profiled_seconds = 0 finally: self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) + else: + self._total_profiled_seconds += 1 \ No newline at end of file From cf15cd069a28777b6348276f90cd593175106727 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 10 Jun 2022 12:53:09 -0400 Subject: [PATCH 05/37] fix: use 60s, not 61s --- ddtrace/profiling/serverless_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 21a3a2d518a..b01a9c1b222 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -16,7 +16,7 @@ class ServerlessScheduler(scheduler.Scheduler): _total_profiled_seconds = attr.ib(default=0) def periodic(self): - if self._total_profiled_seconds > self._interval: + if self._total_profiled_seconds >= self._interval: start_time = compat.monotonic() try: self.flush() From 619154106c10cd621450114241f2e4678683769e Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 10 Jun 2022 16:17:47 -0400 Subject: [PATCH 06/37] feat: test. Back to 60s --- ddtrace/profiling/serverless_scheduler.py | 2 +- tests/profiling/test_serverless_scheduler.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/profiling/test_serverless_scheduler.py diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index b01a9c1b222..ceb27cdc153 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -16,7 +16,7 @@ class ServerlessScheduler(scheduler.Scheduler): _total_profiled_seconds = attr.ib(default=0) def periodic(self): - if self._total_profiled_seconds >= self._interval: + if self._total_profiled_seconds >= 60: start_time = compat.monotonic() try: self.flush() diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py new file mode 100644 index 00000000000..bdd3910425f --- /dev/null +++ b/tests/profiling/test_serverless_scheduler.py @@ -0,0 +1,12 @@ +# -*- encoding: utf-8 -*- + +from ddtrace.profiling import exporter +from ddtrace.profiling import recorder +from ddtrace.profiling import serverless_scheduler + + +def test_periodic(): + r = recorder.Recorder() + s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) + s.periodic() + assert s._total_profiled_seconds == 1 \ No newline at end of file From 80c71330afda1f6665d9f3cd83be4ef1ca31fa3a Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jun 2022 13:11:04 -0400 Subject: [PATCH 07/37] feat: Fix double flushing issue by comparing last export time as well as counting number of profiled seconds --- ddtrace/profiling/serverless_scheduler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index ceb27cdc153..3eb13124162 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -16,11 +16,12 @@ class ServerlessScheduler(scheduler.Scheduler): _total_profiled_seconds = attr.ib(default=0) def periodic(self): - if self._total_profiled_seconds >= 60: + now = compat.time_ns() + if now - self._last_export >= 60 * 1e9 and self._total_profiled_seconds >= 60: + self._total_profiled_seconds = 0 start_time = compat.monotonic() try: self.flush() - self._total_profiled_seconds = 0 finally: self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) else: From 9329c263a1d8f52475280deffd9fd3ffefdf7783 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jun 2022 13:44:00 -0400 Subject: [PATCH 08/37] feat: riot formatter --- ddtrace/profiling/exporter/http.py | 4 +--- ddtrace/profiling/profiler.py | 8 ++++---- ddtrace/profiling/recorder.py | 2 +- ddtrace/profiling/serverless_scheduler.py | 5 +++-- setup.py | 2 +- tests/profiling/test_serverless_scheduler.py | 2 +- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index 5dd11bb5f16..6bdc6505652 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -86,9 +86,7 @@ def __attrs_post_init__(self): ) lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode() if lambda_function_name: - tags.update({ - "functionname": lambda_function_name - }) + tags.update({"functionname": lambda_function_name}) if self.version: tags["version"] = self.version.encode("utf-8") diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index f6f31a19344..7f9f1685e5d 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -18,8 +18,8 @@ from ddtrace.profiling import collector from ddtrace.profiling import exporter from ddtrace.profiling import recorder -from ddtrace.profiling import serverless_scheduler from ddtrace.profiling import scheduler +from ddtrace.profiling import serverless_scheduler from ddtrace.profiling.collector import asyncio from ddtrace.profiling.collector import memalloc from ddtrace.profiling.collector import stack @@ -213,11 +213,11 @@ def __attrs_post_init__(self): lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode() if lambda_function_name: self._scheduler = serverless_scheduler.ServerlessScheduler( - recorder=r, exporters=exporters, before_flush=self._collectors_snapshot + recorder=r, exporters=exporters, before_flush=self._collectors_snapshot ) else: self._scheduler = scheduler.Scheduler( - recorder=r, exporters=exporters, before_flush=self._collectors_snapshot + recorder=r, exporters=exporters, before_flush=self._collectors_snapshot ) self.set_asyncio_event_loop_policy() @@ -290,4 +290,4 @@ def _stop_service( # type: ignore[override] pass for col in reversed(self._collectors): - col.join() \ No newline at end of file + col.join() diff --git a/ddtrace/profiling/recorder.py b/ddtrace/profiling/recorder.py index 7e68b1b9413..3bfc5cb46f0 100644 --- a/ddtrace/profiling/recorder.py +++ b/ddtrace/profiling/recorder.py @@ -93,4 +93,4 @@ def reset(self): with self._events_lock: events = self.events self._reset_events() - return events \ No newline at end of file + return events diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 3eb13124162..f69e512f16a 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -2,9 +2,10 @@ import logging import attr + from ddtrace.internal import compat -from ddtrace.profiling import scheduler from ddtrace.internal.utils import attr as attr_utils +from ddtrace.profiling import scheduler LOG = logging.getLogger(__name__) @@ -25,4 +26,4 @@ def periodic(self): finally: self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) else: - self._total_profiled_seconds += 1 \ No newline at end of file + self._total_profiled_seconds += 1 diff --git a/setup.py b/setup.py index 3f6078f952a..5ac41c5795c 100644 --- a/setup.py +++ b/setup.py @@ -210,7 +210,7 @@ def get_exts_for(name): "bytecode; python_version>='3.8'", ] -extra_compile_args.append('-std=gnu99') +extra_compile_args.append("-std=gnu99") setup( name="ddtrace", description="Datadog APM client library", diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index bdd3910425f..e7ffabf743c 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -9,4 +9,4 @@ def test_periodic(): r = recorder.Recorder() s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) s.periodic() - assert s._total_profiled_seconds == 1 \ No newline at end of file + assert s._total_profiled_seconds == 1 From 5ecd1a25d40acc467a922852f7f56d39e45b8bbc Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jun 2022 14:00:53 -0400 Subject: [PATCH 09/37] fix: _interval should be int, not float --- ddtrace/profiling/serverless_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index f69e512f16a..ccc0fa3b701 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -13,7 +13,7 @@ @attr.s class ServerlessScheduler(scheduler.Scheduler): - _interval = attr.ib(factory=attr_utils.from_env("DD_SERVERLESS_PROFILING_UPLOAD_INTERVAL", 1, float)) + _interval = attr.ib(factory=attr_utils.from_env("DD_SERVERLESS_PROFILING_UPLOAD_INTERVAL", 1, int)) _total_profiled_seconds = attr.ib(default=0) def periodic(self): From d104d70a69396c78c4e5e6a4d5080308c747205b Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jun 2022 14:15:00 -0400 Subject: [PATCH 10/37] feat: remove unneeded .encode() --- ddtrace/profiling/profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 7f9f1685e5d..a000268737e 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -210,7 +210,7 @@ def __attrs_post_init__(self): exporters = self._build_default_exporters() if exporters: - lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode() + lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME") if lambda_function_name: self._scheduler = serverless_scheduler.ServerlessScheduler( recorder=r, exporters=exporters, before_flush=self._collectors_snapshot From cf376dfdb229fac85e9d3c6cbf855b94ac1abd22 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jun 2022 14:30:01 -0400 Subject: [PATCH 11/37] feat: revert setup --- setup.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 5ac41c5795c..eeaef56f4af 100644 --- a/setup.py +++ b/setup.py @@ -210,7 +210,7 @@ def get_exts_for(name): "bytecode; python_version>='3.8'", ] -extra_compile_args.append("-std=gnu99") + setup( name="ddtrace", description="Datadog APM client library", @@ -296,13 +296,11 @@ def get_exts_for(name): "ddtrace.internal._rand", sources=["ddtrace/internal/_rand.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.internal._tagset", sources=["ddtrace/internal/_tagset.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Extension( "ddtrace.internal._encoding", @@ -321,31 +319,26 @@ def get_exts_for(name): "ddtrace.profiling.collector._traceback", sources=["ddtrace/profiling/collector/_traceback.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling._threading", sources=["ddtrace/profiling/_threading.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling.collector._task", sources=["ddtrace/profiling/collector/_task.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling.exporter.pprof", sources=["ddtrace/profiling/exporter/pprof.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.profiling._build", sources=["ddtrace/profiling/_build.pyx"], language="c", - extra_compile_args=extra_compile_args, ), Cython.Distutils.Extension( "ddtrace.appsec._ddwaf", From 0978935b19dac4795f89c79e7f761de790e33472 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jun 2022 14:39:28 -0400 Subject: [PATCH 12/37] feat: Move encode to after nil check --- ddtrace/profiling/exporter/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index 6bdc6505652..29f18924189 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -84,9 +84,9 @@ def __attrs_post_init__(self): "profiler_version": ddtrace.__version__.encode("ascii"), } ) - lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME").encode() + lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME") if lambda_function_name: - tags.update({"functionname": lambda_function_name}) + tags.update({"functionname": lambda_function_name.encode()}) if self.version: tags["version"] = self.version.encode("utf-8") From b8c44600810aefab6b70f23663bb7226db94bce5 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 27 Jun 2022 14:47:15 -0400 Subject: [PATCH 13/37] feat: Guard against unset _last_export. Add tests for overriden periodic function. --- ddtrace/profiling/serverless_scheduler.py | 4 +++- tests/profiling/test_serverless_scheduler.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index ccc0fa3b701..2b979dc4443 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -18,7 +18,9 @@ class ServerlessScheduler(scheduler.Scheduler): def periodic(self): now = compat.time_ns() - if now - self._last_export >= 60 * 1e9 and self._total_profiled_seconds >= 60: + # Guard against _last_export not being set + last_export = self._last_export or compat.time_ns() + if now - last_export >= 60 * 1e9 and self._total_profiled_seconds >= 60: self._total_profiled_seconds = 0 start_time = compat.monotonic() try: diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index e7ffabf743c..1602d81a987 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -3,10 +3,11 @@ from ddtrace.profiling import exporter from ddtrace.profiling import recorder from ddtrace.profiling import serverless_scheduler - +from ddtrace.internal import compat def test_periodic(): r = recorder.Recorder() s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) + s._last_export = compat.time_ns() s.periodic() assert s._total_profiled_seconds == 1 From 4afe52c112ec6242138b3355980494619bb1e13c Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 27 Jun 2022 14:54:38 -0400 Subject: [PATCH 14/37] feat: formatting --- tests/profiling/test_serverless_scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index 1602d81a987..3a3852a6eb8 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -1,9 +1,10 @@ # -*- encoding: utf-8 -*- +from ddtrace.internal import compat from ddtrace.profiling import exporter from ddtrace.profiling import recorder from ddtrace.profiling import serverless_scheduler -from ddtrace.internal import compat + def test_periodic(): r = recorder.Recorder() From 754367c40866ba96cfd797430d395a105ce065ba Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 27 Jun 2022 15:12:05 -0400 Subject: [PATCH 15/37] feat: release notes --- .../notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml diff --git a/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml b/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml new file mode 100644 index 00000000000..35fae65fd1b --- /dev/null +++ b/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds serverless_scheduler, which flushes data after 60 calls to `periodic`, which occur every second. From d3e5a4c637c6eb43734a07ab4c894feb416196dc Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 27 Jun 2022 15:35:09 -0400 Subject: [PATCH 16/37] feat: Add serverless to spelling list --- docs/spelling_wordlist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 90ad67bc9a9..8d64c69c9d6 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -186,3 +186,5 @@ yaaredis Kinesis AppSec libddwaf +Serverless +serverless From 79c2f23302dd16264f1ce4df8f8bd59e8f423303 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 18 Jul 2022 11:45:36 -0400 Subject: [PATCH 17/37] feat: Incorporate feedback from JD --- ddtrace/profiling/exporter/http.py | 3 --- ddtrace/profiling/profiler.py | 4 ++++ ddtrace/profiling/serverless_scheduler.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index 29f18924189..d3405200a63 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -84,9 +84,6 @@ def __attrs_post_init__(self): "profiler_version": ddtrace.__version__.encode("ascii"), } ) - lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME") - if lambda_function_name: - tags.update({"functionname": lambda_function_name.encode()}) if self.version: tags["version"] = self.version.encode("utf-8") diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index a000268737e..f4c87e6076d 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -45,6 +45,7 @@ class Profiler(object): def __init__(self, *args, **kwargs): self._profiler = _ProfilerInstance(*args, **kwargs) + self.lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", type=Optional[str]) def start(self, stop_on_exit=True, profile_children=True): """Start the profiler. @@ -166,6 +167,9 @@ def _build_default_exporters(self): # to the agent base path. endpoint_path = "profiling/v1/input" + if self.lambda_function_name: + self.tags.update({"functionname": self.lambda_function_name.encode()}) + return [ http.PprofHTTPExporter( service=self.service, diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 2b979dc4443..295b2216b7e 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -13,7 +13,7 @@ @attr.s class ServerlessScheduler(scheduler.Scheduler): - _interval = attr.ib(factory=attr_utils.from_env("DD_SERVERLESS_PROFILING_UPLOAD_INTERVAL", 1, int)) + _interval = attr.ib(1) _total_profiled_seconds = attr.ib(default=0) def periodic(self): @@ -28,4 +28,4 @@ def periodic(self): finally: self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) else: - self._total_profiled_seconds += 1 + self._total_profiled_seconds += self._interval From 94cd7f2a3f0ac6e8d062385e0d96c80c4f31ddbb Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 18 Jul 2022 14:23:31 -0400 Subject: [PATCH 18/37] feat: CR feedback from JD --- ddtrace/profiling/profiler.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index f4c87e6076d..c28e5b09e09 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -45,7 +45,6 @@ class Profiler(object): def __init__(self, *args, **kwargs): self._profiler = _ProfilerInstance(*args, **kwargs) - self.lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", type=Optional[str]) def start(self, stop_on_exit=True, profile_children=True): """Start the profiler. @@ -134,6 +133,7 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) _scheduler = attr.ib(init=False, default=None) + _lambda_function_name = attr.ib(factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]) ENDPOINT_TEMPLATE = "https://intake.profile.{}" @@ -167,8 +167,8 @@ def _build_default_exporters(self): # to the agent base path. endpoint_path = "profiling/v1/input" - if self.lambda_function_name: - self.tags.update({"functionname": self.lambda_function_name.encode()}) + if self._lambda_function_name: + self.tags.update({"functionname": self._lambda_function_name.encode()}) return [ http.PprofHTTPExporter( @@ -214,8 +214,7 @@ def __attrs_post_init__(self): exporters = self._build_default_exporters() if exporters: - lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME") - if lambda_function_name: + if self._lambda_function_name: self._scheduler = serverless_scheduler.ServerlessScheduler( recorder=r, exporters=exporters, before_flush=self._collectors_snapshot ) From 3e5c9e44ac8f7eba3b7585b372b7097a1068585d Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 18 Jul 2022 14:29:01 -0400 Subject: [PATCH 19/37] lint: fix --- ddtrace/profiling/profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index c28e5b09e09..bc871dd57c3 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -133,7 +133,7 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) _scheduler = attr.ib(init=False, default=None) - _lambda_function_name = attr.ib(factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]) + _lambda_function_name = attr.ib(factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]) ENDPOINT_TEMPLATE = "https://intake.profile.{}" From 811cb04216e729dd6f9b43282fe30716b1e8373d Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 18 Jul 2022 14:33:32 -0400 Subject: [PATCH 20/37] feat: remove unused import --- ddtrace/profiling/serverless_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 295b2216b7e..c1a6f5517c0 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -4,7 +4,6 @@ import attr from ddtrace.internal import compat -from ddtrace.internal.utils import attr as attr_utils from ddtrace.profiling import scheduler From 1ab1ef397db3fbc2aa08f491db7fc8f8f2eccf92 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 18 Jul 2022 14:55:23 -0400 Subject: [PATCH 21/37] feat: Human-readable release note --- .../notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml b/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml index 35fae65fd1b..0059f160364 100644 --- a/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml +++ b/releasenotes/notes/feat-serverless-scheduler-31a819bc9eb4f332.yaml @@ -1,4 +1,4 @@ --- features: - | - Adds serverless_scheduler, which flushes data after 60 calls to `periodic`, which occur every second. + Adds support for Lambda profiling, which can be enabled by starting the profiler outside of the handler (on cold start). From baadc76effdffe3e5d60e5d4293376fa7daa1b9b Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 2 Aug 2022 13:46:36 -0400 Subject: [PATCH 22/37] feat: CR feedback to remove unneeded checks on periodic calls and reduce to once every minute. Test changes to match --- ddtrace/profiling/profiler.py | 9 ++++----- ddtrace/profiling/serverless_scheduler.py | 19 ++++++------------- tests/profiling/test_serverless_scheduler.py | 9 +++++---- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index bc871dd57c3..ef770558fe7 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -133,7 +133,7 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) _scheduler = attr.ib(init=False, default=None) - _lambda_function_name = attr.ib(factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]) + _lambda_function_name = attr.ib(init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]) ENDPOINT_TEMPLATE = "https://intake.profile.{}" @@ -215,11 +215,10 @@ def __attrs_post_init__(self): if exporters: if self._lambda_function_name: - self._scheduler = serverless_scheduler.ServerlessScheduler( - recorder=r, exporters=exporters, before_flush=self._collectors_snapshot - ) + scheduler_class = serverless_scheduler.ServerlessScheduler else: - self._scheduler = scheduler.Scheduler( + scheduler_class = scheduler.Scheduler + self._scheduler = scheduler_class( recorder=r, exporters=exporters, before_flush=self._collectors_snapshot ) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index c1a6f5517c0..5a91a996495 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -5,26 +5,19 @@ from ddtrace.internal import compat from ddtrace.profiling import scheduler - +from ddtrace.internal.utils import attr as attr_utils LOG = logging.getLogger(__name__) @attr.s class ServerlessScheduler(scheduler.Scheduler): - _interval = attr.ib(1) - _total_profiled_seconds = attr.ib(default=0) + _interval = attr.ib(factory=attr_utils.from_env("DD_PROFILING_UPLOAD_INTERVAL", 60.0, float)) + _init_time = compat.time_ns() def periodic(self): now = compat.time_ns() # Guard against _last_export not being set - last_export = self._last_export or compat.time_ns() - if now - last_export >= 60 * 1e9 and self._total_profiled_seconds >= 60: - self._total_profiled_seconds = 0 - start_time = compat.monotonic() - try: - self.flush() - finally: - self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) - else: - self._total_profiled_seconds += self._interval + last_export = self._last_export or self._init_time + if now - last_export >= int(self._interval) * 1e9: + super(ServerlessScheduler, self).periodic() \ No newline at end of file diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index 3a3852a6eb8..727f81dfc74 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -1,14 +1,15 @@ # -*- encoding: utf-8 -*- +import mock from ddtrace.internal import compat from ddtrace.profiling import exporter from ddtrace.profiling import recorder from ddtrace.profiling import serverless_scheduler - -def test_periodic(): +@mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") +def test_periodic(mock_periodic): r = recorder.Recorder() s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) - s._last_export = compat.time_ns() + s._last_export = compat.time_ns() - 65 * 1e9 s.periodic() - assert s._total_profiled_seconds == 1 + mock_periodic.assert_called() \ No newline at end of file From c4ed52ea4a88ac2baf586ff7f54b282cf79e0dff Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 2 Aug 2022 13:51:52 -0400 Subject: [PATCH 23/37] feat: riot --- ddtrace/profiling/profiler.py | 8 ++++---- ddtrace/profiling/serverless_scheduler.py | 5 +++-- tests/profiling/test_serverless_scheduler.py | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index ef770558fe7..8b7f8bd030e 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -133,7 +133,9 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) _scheduler = attr.ib(init=False, default=None) - _lambda_function_name = attr.ib(init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]) + _lambda_function_name = attr.ib( + init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str] + ) ENDPOINT_TEMPLATE = "https://intake.profile.{}" @@ -218,9 +220,7 @@ def __attrs_post_init__(self): scheduler_class = serverless_scheduler.ServerlessScheduler else: scheduler_class = scheduler.Scheduler - self._scheduler = scheduler_class( - recorder=r, exporters=exporters, before_flush=self._collectors_snapshot - ) + self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) self.set_asyncio_event_loop_policy() diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 5a91a996495..5ed709bef70 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -4,8 +4,9 @@ import attr from ddtrace.internal import compat -from ddtrace.profiling import scheduler from ddtrace.internal.utils import attr as attr_utils +from ddtrace.profiling import scheduler + LOG = logging.getLogger(__name__) @@ -20,4 +21,4 @@ def periodic(self): # Guard against _last_export not being set last_export = self._last_export or self._init_time if now - last_export >= int(self._interval) * 1e9: - super(ServerlessScheduler, self).periodic() \ No newline at end of file + super(ServerlessScheduler, self).periodic() diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index 727f81dfc74..a6e5029d9cb 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -6,10 +6,11 @@ from ddtrace.profiling import recorder from ddtrace.profiling import serverless_scheduler + @mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") def test_periodic(mock_periodic): r = recorder.Recorder() s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) s._last_export = compat.time_ns() - 65 * 1e9 s.periodic() - mock_periodic.assert_called() \ No newline at end of file + mock_periodic.assert_called() From e6845bfb155bde0660db4d2dc183ef6baee57d94 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 2 Aug 2022 15:15:22 -0400 Subject: [PATCH 24/37] wip: skipping assignment type check, going to ask JD for help --- ddtrace/profiling/profiler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 8b7f8bd030e..dbf0a2fa4c0 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -4,6 +4,7 @@ import typing from typing import List from typing import Optional +from typing import Union import attr @@ -132,7 +133,7 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) - _scheduler = attr.ib(init=False, default=None) + _scheduler = attr.ib(init=False, default=None, type=Union[scheduler.Scheduler, serverless_scheduler.ServerlessScheduler]) _lambda_function_name = attr.ib( init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str] ) @@ -219,7 +220,7 @@ def __attrs_post_init__(self): if self._lambda_function_name: scheduler_class = serverless_scheduler.ServerlessScheduler else: - scheduler_class = scheduler.Scheduler + scheduler_class = scheduler.Scheduler # type: ignore[assignment] self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) self.set_asyncio_event_loop_policy() From 3df593314e113db7d5c780a0af3eb2f8719ba72e Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 2 Aug 2022 15:23:48 -0400 Subject: [PATCH 25/37] feat: fmt --- ddtrace/profiling/profiler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index dbf0a2fa4c0..13a6dd9b63e 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -133,7 +133,9 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) - _scheduler = attr.ib(init=False, default=None, type=Union[scheduler.Scheduler, serverless_scheduler.ServerlessScheduler]) + _scheduler = attr.ib( + init=False, default=None, type=Union[scheduler.Scheduler, serverless_scheduler.ServerlessScheduler] + ) _lambda_function_name = attr.ib( init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str] ) @@ -220,7 +222,7 @@ def __attrs_post_init__(self): if self._lambda_function_name: scheduler_class = serverless_scheduler.ServerlessScheduler else: - scheduler_class = scheduler.Scheduler # type: ignore[assignment] + scheduler_class = scheduler.Scheduler # type: ignore[assignment] self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) self.set_asyncio_event_loop_policy() From 910971878c6946581446e526a9785fa8bbce3c32 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 3 Aug 2022 18:32:08 -0400 Subject: [PATCH 26/37] feat: re-add counter check to ensure we don't flush early. --- ddtrace/profiling/profiler.py | 6 +++--- ddtrace/profiling/serverless_scheduler.py | 8 ++++++-- tests/profiling/test_serverless_scheduler.py | 5 +++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 13a6dd9b63e..e185f8c0722 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -220,9 +220,9 @@ def __attrs_post_init__(self): if exporters: if self._lambda_function_name: - scheduler_class = serverless_scheduler.ServerlessScheduler + scheduler_class = serverless_scheduler.ServerlessScheduler # type: Union[type[scheduler.Scheduler], type[serverless_scheduler.ServerlessScheduler]] else: - scheduler_class = scheduler.Scheduler # type: ignore[assignment] + scheduler_class = scheduler.Scheduler self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) self.set_asyncio_event_loop_policy() @@ -295,4 +295,4 @@ def _stop_service( # type: ignore[override] pass for col in reversed(self._collectors): - col.join() + col.join() \ No newline at end of file diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 5ed709bef70..75d462b238d 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -13,12 +13,16 @@ @attr.s class ServerlessScheduler(scheduler.Scheduler): - _interval = attr.ib(factory=attr_utils.from_env("DD_PROFILING_UPLOAD_INTERVAL", 60.0, float)) + _interval = attr.ib(default=1.0, type=float) + _profiled_intervals = attr.ib(default=0) _init_time = compat.time_ns() def periodic(self): now = compat.time_ns() # Guard against _last_export not being set last_export = self._last_export or self._init_time - if now - last_export >= int(self._interval) * 1e9: + if (now - last_export >= int(self._interval) * 1e9) and (self._profiled_intervals >= 60): + self._profiled_intervals = 0 super(ServerlessScheduler, self).periodic() + else: + self._profiled_intervals += 1 \ No newline at end of file diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index a6e5029d9cb..308fd5720b2 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -4,13 +4,14 @@ from ddtrace.internal import compat from ddtrace.profiling import exporter from ddtrace.profiling import recorder -from ddtrace.profiling import serverless_scheduler +from ddtrace.profiling import external_scheduler @mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") def test_periodic(mock_periodic): r = recorder.Recorder() - s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) + s = external_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) s._last_export = compat.time_ns() - 65 * 1e9 + s._profiled_intervals = 65 s.periodic() mock_periodic.assert_called() From 528291676571d752c31162118a2d9c61090429bc Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 3 Aug 2022 18:38:24 -0400 Subject: [PATCH 27/37] feat: riot fmt --- ddtrace/profiling/profiler.py | 6 ++++-- ddtrace/profiling/serverless_scheduler.py | 2 +- tests/profiling/test_serverless_scheduler.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index e185f8c0722..67527d7da54 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -220,7 +220,9 @@ def __attrs_post_init__(self): if exporters: if self._lambda_function_name: - scheduler_class = serverless_scheduler.ServerlessScheduler # type: Union[type[scheduler.Scheduler], type[serverless_scheduler.ServerlessScheduler]] + scheduler_class = ( + serverless_scheduler.ServerlessScheduler + ) # type: Union[type[scheduler.Scheduler], type[serverless_scheduler.ServerlessScheduler]] else: scheduler_class = scheduler.Scheduler self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) @@ -295,4 +297,4 @@ def _stop_service( # type: ignore[override] pass for col in reversed(self._collectors): - col.join() \ No newline at end of file + col.join() diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 75d462b238d..7a3b21fee37 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -25,4 +25,4 @@ def periodic(self): self._profiled_intervals = 0 super(ServerlessScheduler, self).periodic() else: - self._profiled_intervals += 1 \ No newline at end of file + self._profiled_intervals += 1 diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index 308fd5720b2..c32cd24cab6 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -3,8 +3,8 @@ from ddtrace.internal import compat from ddtrace.profiling import exporter -from ddtrace.profiling import recorder from ddtrace.profiling import external_scheduler +from ddtrace.profiling import recorder @mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") From ea30d06b7581dd4ff0edf73721a833420732fe40 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 3 Aug 2022 18:42:37 -0400 Subject: [PATCH 28/37] feat: remove unused import --- ddtrace/profiling/serverless_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 7a3b21fee37..23a36e427f2 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -4,7 +4,6 @@ import attr from ddtrace.internal import compat -from ddtrace.internal.utils import attr as attr_utils from ddtrace.profiling import scheduler From 77c133bf509b0797305db2a88ab66731f1329065 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 3 Aug 2022 19:14:53 -0400 Subject: [PATCH 29/37] feat: fix bad rename --- tests/profiling/test_serverless_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index c32cd24cab6..e5bb393831c 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -3,14 +3,14 @@ from ddtrace.internal import compat from ddtrace.profiling import exporter -from ddtrace.profiling import external_scheduler +from ddtrace.profiling import serverless_scheduler from ddtrace.profiling import recorder @mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") def test_periodic(mock_periodic): r = recorder.Recorder() - s = external_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) + s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) s._last_export = compat.time_ns() - 65 * 1e9 s._profiled_intervals = 65 s.periodic() From b64b3ebe9f0be1513a8724a2e169ef8a9a445201 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 3 Aug 2022 19:26:26 -0400 Subject: [PATCH 30/37] feat: more format --- tests/profiling/test_serverless_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py index e5bb393831c..2d489bc127d 100644 --- a/tests/profiling/test_serverless_scheduler.py +++ b/tests/profiling/test_serverless_scheduler.py @@ -3,8 +3,8 @@ from ddtrace.internal import compat from ddtrace.profiling import exporter -from ddtrace.profiling import serverless_scheduler from ddtrace.profiling import recorder +from ddtrace.profiling import serverless_scheduler @mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") From aededd57eb463f21e85f6dbf50fc8e9b1ddfd00c Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 4 Aug 2022 10:39:35 -0400 Subject: [PATCH 31/37] feat: Override reset interval after flushing --- ddtrace/profiling/serverless_scheduler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py index 23a36e427f2..08f13f55133 100644 --- a/ddtrace/profiling/serverless_scheduler.py +++ b/ddtrace/profiling/serverless_scheduler.py @@ -14,14 +14,12 @@ class ServerlessScheduler(scheduler.Scheduler): _interval = attr.ib(default=1.0, type=float) _profiled_intervals = attr.ib(default=0) - _init_time = compat.time_ns() def periodic(self): now = compat.time_ns() - # Guard against _last_export not being set - last_export = self._last_export or self._init_time - if (now - last_export >= int(self._interval) * 1e9) and (self._profiled_intervals >= 60): + if (now - self._last_export) >= 60 * 1e9 and self._profiled_intervals >= 60: self._profiled_intervals = 0 super(ServerlessScheduler, self).periodic() + self.interval = 1.0 else: self._profiled_intervals += 1 From e600cbe6db3feb619d2043df1d21a8296292cd87 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 5 Aug 2022 14:55:06 +0200 Subject: [PATCH 32/37] try fixing typing --- ddtrace/profiling/profiler.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 67527d7da54..df094846ea1 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -4,7 +4,6 @@ import typing from typing import List from typing import Optional -from typing import Union import attr @@ -134,7 +133,9 @@ class _ProfilerInstance(service.Service): _recorder = attr.ib(init=False, default=None) _collectors = attr.ib(init=False, default=None) _scheduler = attr.ib( - init=False, default=None, type=Union[scheduler.Scheduler, serverless_scheduler.ServerlessScheduler] + init=False, + default=None, + type=scheduler.Scheduler, ) _lambda_function_name = attr.ib( init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str] @@ -219,12 +220,10 @@ def __attrs_post_init__(self): exporters = self._build_default_exporters() if exporters: - if self._lambda_function_name: - scheduler_class = ( - serverless_scheduler.ServerlessScheduler - ) # type: Union[type[scheduler.Scheduler], type[serverless_scheduler.ServerlessScheduler]] - else: + if not self._lambda_function_name: scheduler_class = scheduler.Scheduler + else: + scheduler_class = serverless_scheduler.ServerlessScheduler self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) self.set_asyncio_event_loop_policy() From 662f681c25bb81adc66778ff95f46bdbf0703764 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 5 Aug 2022 15:01:28 +0200 Subject: [PATCH 33/37] move serverless into scheduler --- ddtrace/profiling/profiler.py | 1 - ddtrace/profiling/scheduler.py | 23 ++++++++++++++++++ ddtrace/profiling/serverless_scheduler.py | 25 -------------------- tests/profiling/test_scheduler.py | 13 ++++++++++ tests/profiling/test_serverless_scheduler.py | 17 ------------- 5 files changed, 36 insertions(+), 43 deletions(-) delete mode 100644 ddtrace/profiling/serverless_scheduler.py delete mode 100644 tests/profiling/test_serverless_scheduler.py diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index df094846ea1..749e8904bd8 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -19,7 +19,6 @@ from ddtrace.profiling import exporter from ddtrace.profiling import recorder from ddtrace.profiling import scheduler -from ddtrace.profiling import serverless_scheduler from ddtrace.profiling.collector import asyncio from ddtrace.profiling.collector import memalloc from ddtrace.profiling.collector import stack diff --git a/ddtrace/profiling/scheduler.py b/ddtrace/profiling/scheduler.py index 323f4fba24e..93c61972694 100644 --- a/ddtrace/profiling/scheduler.py +++ b/ddtrace/profiling/scheduler.py @@ -65,3 +65,26 @@ def periodic(self): self.flush() finally: self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) + + +@attr.s +class ServerlessScheduler(Scheduler): + """Serverless scheduler that works on, e.g., AWS Lambda. + + The idea with this scheduler is to not sleep 60s, but to sleep 1s and flush out profiles after 60 sleeping period. + As the service can be frozen a few seconds after flushing out a profile, we want to make sure the next flush is not + > 60s later, but after at least 60 periods of 1s. + + """ + _interval = attr.ib(default=1.0, type=float) + _profiled_intervals = attr.ib(init=False, default=0) + + def periodic(self): + now = compat.time_ns() + if (now - self._last_export) >= 60 * 1e9 and self._profiled_intervals >= 60: + self._profiled_intervals = 0 + super(ServerlessScheduler, self).periodic() + # Override interval so it's always 1 + self.interval = 1.0 + else: + self._profiled_intervals += 1 diff --git a/ddtrace/profiling/serverless_scheduler.py b/ddtrace/profiling/serverless_scheduler.py deleted file mode 100644 index 08f13f55133..00000000000 --- a/ddtrace/profiling/serverless_scheduler.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- encoding: utf-8 -*- -import logging - -import attr - -from ddtrace.internal import compat -from ddtrace.profiling import scheduler - - -LOG = logging.getLogger(__name__) - - -@attr.s -class ServerlessScheduler(scheduler.Scheduler): - _interval = attr.ib(default=1.0, type=float) - _profiled_intervals = attr.ib(default=0) - - def periodic(self): - now = compat.time_ns() - if (now - self._last_export) >= 60 * 1e9 and self._profiled_intervals >= 60: - self._profiled_intervals = 0 - super(ServerlessScheduler, self).periodic() - self.interval = 1.0 - else: - self._profiled_intervals += 1 diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 49effaa4beb..ec353caa692 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -1,6 +1,9 @@ # -*- encoding: utf-8 -*- import logging +import mock + +from ddtrace.internal import compat from ddtrace.profiling import event from ddtrace.profiling import exporter from ddtrace.profiling import recorder @@ -54,3 +57,13 @@ def call_me(): assert caplog.record_tuples == [ (("ddtrace.profiling.scheduler", logging.ERROR, "Scheduler before_flush hook failed")) ] + + +@mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") +def test_serverless_periodic(mock_periodic): + r = recorder.Recorder() + s = scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) + s._last_export = compat.time_ns() - 65 * 1e9 + s._profiled_intervals = 65 + s.periodic() + mock_periodic.assert_called() diff --git a/tests/profiling/test_serverless_scheduler.py b/tests/profiling/test_serverless_scheduler.py deleted file mode 100644 index 2d489bc127d..00000000000 --- a/tests/profiling/test_serverless_scheduler.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- encoding: utf-8 -*- -import mock - -from ddtrace.internal import compat -from ddtrace.profiling import exporter -from ddtrace.profiling import recorder -from ddtrace.profiling import serverless_scheduler - - -@mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") -def test_periodic(mock_periodic): - r = recorder.Recorder() - s = serverless_scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) - s._last_export = compat.time_ns() - 65 * 1e9 - s._profiled_intervals = 65 - s.periodic() - mock_periodic.assert_called() From 528e325685e21eaae1e4855fe7e30429cec8f049 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 5 Aug 2022 15:49:38 +0200 Subject: [PATCH 34/37] use some var --- ddtrace/profiling/profiler.py | 2 +- ddtrace/profiling/scheduler.py | 23 ++++++++++++++++------- tests/profiling/test_scheduler.py | 8 +++++++- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 749e8904bd8..e5de65be565 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -222,7 +222,7 @@ def __attrs_post_init__(self): if not self._lambda_function_name: scheduler_class = scheduler.Scheduler else: - scheduler_class = serverless_scheduler.ServerlessScheduler + scheduler_class = scheduler.ServerlessScheduler self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot) self.set_asyncio_event_loop_policy() diff --git a/ddtrace/profiling/scheduler.py b/ddtrace/profiling/scheduler.py index 93c61972694..9bf2d3d139a 100644 --- a/ddtrace/profiling/scheduler.py +++ b/ddtrace/profiling/scheduler.py @@ -76,15 +76,24 @@ class ServerlessScheduler(Scheduler): > 60s later, but after at least 60 periods of 1s. """ - _interval = attr.ib(default=1.0, type=float) + + # We force this interval everywhere + FORCED_INTERVAL = 1.0 + FLUSH_AFTER_INTERVALS = 60.0 + + _interval = attr.ib(default=FORCED_INTERVAL, type=float) _profiled_intervals = attr.ib(init=False, default=0) def periodic(self): - now = compat.time_ns() - if (now - self._last_export) >= 60 * 1e9 and self._profiled_intervals >= 60: - self._profiled_intervals = 0 - super(ServerlessScheduler, self).periodic() - # Override interval so it's always 1 - self.interval = 1.0 + # Check both the number of intervals and time frame to be sure we don't flush, e.g., empty profiles + if self._profiled_intervals >= self.FLUSH_AFTER_INTERVALS and (compat.time_ns() - self._last_export) >= ( + self.FORCED_INTERVAL * self.FLUSH_AFTER_INTERVALS + ): + try: + super(ServerlessScheduler, self).periodic() + finally: + # Override interval so it's always back to the value we n + self.interval = self.FORCED_INTERVAL + self._profiled_intervals = 0 else: self._profiled_intervals += 1 diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index ec353caa692..c84b42747b6 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -63,7 +63,13 @@ def call_me(): def test_serverless_periodic(mock_periodic): r = recorder.Recorder() s = scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) - s._last_export = compat.time_ns() - 65 * 1e9 + # Fake start() + s._last_export = compat.time_ns() + s.periodic() + assert s._profiled_intervals == 1 + mock_periodic.assert_not_called() s._profiled_intervals = 65 s.periodic() + assert s._profiled_intervals == 0 + assert s.interval == 1 mock_periodic.assert_called() From 6e2ab1d42257a4cfabeac45d81b10a5583fe0230 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 5 Aug 2022 16:08:29 +0200 Subject: [PATCH 35/37] test(serverless): add profiler test --- tests/profiling/test_profiler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/profiling/test_profiler.py b/tests/profiling/test_profiler.py index 7da3d630c2d..39dd413728d 100644 --- a/tests/profiling/test_profiler.py +++ b/tests/profiling/test_profiler.py @@ -9,6 +9,7 @@ from ddtrace.profiling import event from ddtrace.profiling import exporter from ddtrace.profiling import profiler +from ddtrace.profiling import scheduler from ddtrace.profiling.collector import asyncio from ddtrace.profiling.collector import memalloc from ddtrace.profiling.collector import stack @@ -378,3 +379,11 @@ def test_default_collectors(): else: assert any(isinstance(c, asyncio.AsyncioLockCollector) for c in p._profiler._collectors) p.stop(flush=False) + + +def test_profiler_serverless(monkeypatch): + # type: (...) -> None + monkeypatch.setenv("AWS_LAMBDA_FUNCTION_NAME", "foobar") + p = profiler.Profiler() + assert isinstance(p._scheduler, scheduler.ServerlessScheduler) + assert p.tags["functionname"] == b"foobar" From 57eef9c852a34810cc857a1ac49e16c709bb534f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 5 Aug 2022 17:08:17 +0200 Subject: [PATCH 36/37] fix profile test case --- tests/profiling/test_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index c84b42747b6..483ec643527 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -68,6 +68,7 @@ def test_serverless_periodic(mock_periodic): s.periodic() assert s._profiled_intervals == 1 mock_periodic.assert_not_called() + s._last_export = compat.time_ns() - 65 s._profiled_intervals = 65 s.periodic() assert s._profiled_intervals == 0 From fc2c22b9bbaa94ac0532bc051da7019972dc2f52 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 8 Aug 2022 06:56:22 -0400 Subject: [PATCH 37/37] feat: more CR feedback. Flipping implicit none checks to explicit, using utf-8 encoding explicitly --- ddtrace/profiling/profiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index e5de65be565..918d0f74889 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -172,8 +172,8 @@ def _build_default_exporters(self): # to the agent base path. endpoint_path = "profiling/v1/input" - if self._lambda_function_name: - self.tags.update({"functionname": self._lambda_function_name.encode()}) + if self._lambda_function_name is not None: + self.tags.update({"functionname": self._lambda_function_name.encode("utf-8")}) return [ http.PprofHTTPExporter( @@ -219,7 +219,7 @@ def __attrs_post_init__(self): exporters = self._build_default_exporters() if exporters: - if not self._lambda_function_name: + if self._lambda_function_name is None: scheduler_class = scheduler.Scheduler else: scheduler_class = scheduler.ServerlessScheduler