diff --git a/Makefile b/Makefile index 5b5ca4d76..2131ffbe8 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ package-lock.json: package.json touch $@ test: - DJANGO_SETTINGS_MODULE=tests.settings \ + DB_BACKEND=sqlite3 DB_NAME=":memory:" DJANGO_SETTINGS_MODULE=tests.settings \ python -m django test $${TEST_ARGS:-tests} test_selenium: diff --git a/debug_toolbar/db_store.py b/debug_toolbar/db_store.py new file mode 100644 index 000000000..3babb92f9 --- /dev/null +++ b/debug_toolbar/db_store.py @@ -0,0 +1,50 @@ +from debug_toolbar import store +from debug_toolbar.models import PanelStore, ToolbarStore + + +class DBStore(store.BaseStore): + @classmethod + def ids(cls): + return ( + ToolbarStore.objects.using("debug_toolbar") + .values_list("key", flat=True) + .order_by("created") + ) + + @classmethod + def exists(cls, store_id): + return ToolbarStore.objects.using("debug_toolbar").filter(key=store_id).exists() + + @classmethod + def set(cls, store_id): + _, created = ToolbarStore.objects.using("debug_toolbar").get_or_create( + key=store_id + ) + if ( + created + and ToolbarStore.objects.using("debug_toolbar").all().count() + > cls.config["RESULTS_CACHE_SIZE"] + ): + ToolbarStore.objects.using("debug_toolbar").earliest("created").delete() + + @classmethod + def delete(cls, store_id): + ToolbarStore.objects.using("debug_toolbar").filter(key=store_id).delete() + + @classmethod + def save_panel(cls, store_id, panel_id, stats=None): + toolbar, _ = ToolbarStore.objects.using("debug_toolbar").get_or_create( + key=store_id + ) + toolbar.panelstore_set.update_or_create( + panel=panel_id, defaults={"data": store.serialize(stats)} + ) + + @classmethod + def panel(cls, store_id, panel_id): + panel = ( + PanelStore.objects.using("debug_toolbar") + .filter(toolbar__key=store_id, panel=panel_id) + .first() + ) + return {} if not panel else store.deserialize(panel.data) diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py new file mode 100644 index 000000000..ec6c8c770 --- /dev/null +++ b/debug_toolbar/migrations/0001_initial.py @@ -0,0 +1,58 @@ +# Generated by Django 3.1.5 on 2021-01-09 17:02 +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="ToolbarStore", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("key", models.CharField(max_length=64, unique=True)), + ], + ), + migrations.CreateModel( + name="PanelStore", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("panel", models.CharField(max_length=128)), + ("data", models.TextField()), + ( + "toolbar", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="debug_toolbar.toolbarstore", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="panelstore", + constraint=models.UniqueConstraint( + fields=("toolbar", "panel"), name="unique_toolbar_panel" + ), + ), + ] diff --git a/debug_toolbar/migrations/__init__.py b/debug_toolbar/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py new file mode 100644 index 000000000..1adf0f32e --- /dev/null +++ b/debug_toolbar/models.py @@ -0,0 +1,13 @@ +from django.db import models + + +class ToolbarStore(models.Model): + created = models.DateTimeField(auto_now_add=True) + key = models.CharField(max_length=64, unique=True) + + +class PanelStore(models.Model): + created = models.DateTimeField(auto_now_add=True) + toolbar = models.ForeignKey(ToolbarStore, on_delete=models.CASCADE) + panel = models.CharField(max_length=128) + data = models.TextField() diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index 8fd433c63..2f0c6dbe1 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -1,6 +1,7 @@ from django.template.loader import render_to_string from debug_toolbar import settings as dt_settings +from debug_toolbar.store import get_store from debug_toolbar.utils import get_name_from_obj @@ -37,7 +38,13 @@ def enabled(self): else: default = "on" # The user's cookies should override the default value - return self.toolbar.request.COOKIES.get("djdt" + self.panel_id, default) == "on" + if self.toolbar.request is not None: + return ( + self.toolbar.request.COOKIES.get("djdt" + self.panel_id, default) + == "on" + ) + else: + return bool(get_store().panel(self.toolbar.store_id, self.panel_id)) # Titles and content @@ -150,6 +157,24 @@ def disable_instrumentation(self): # Store and retrieve stats (shared between panels for no good reason) + @classmethod + def deserialize_stats(cls, data): + """ + Deserialize stats coming from the store. + + Provided to support future store mechanisms overriding a panel's content. + """ + return data + + @classmethod + def serialize_stats(cls, stats): + """ + Serialize stats for the store. + + Provided to support future store mechanisms overriding a panel's content. + """ + return stats + def record_stats(self, stats): """ Store data gathered by the panel. ``stats`` is a :class:`dict`. @@ -157,12 +182,17 @@ def record_stats(self, stats): Each call to ``record_stats`` updates the statistics dictionary. """ self.toolbar.stats.setdefault(self.panel_id, {}).update(stats) + get_store().save_panel( + self.toolbar.store_id, self.panel_id, self.serialize_stats(stats) + ) def get_stats(self): """ Access data stored by the panel. Returns a :class:`dict`. """ - return self.toolbar.stats.get(self.panel_id, {}) + return self.deserialize_stats( + get_store().panel(self.toolbar.store_id, self.panel_id) + ) def record_server_timing(self, key, title, value): """ diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py index 41063c573..32dd0ba62 100644 --- a/debug_toolbar/panels/cache.py +++ b/debug_toolbar/panels/cache.py @@ -236,7 +236,7 @@ def _store_call_info( "kwargs": kwargs, "trace": render_stacktrace(trace), "template_info": template_info, - "backend": backend, + "backend": str(backend), } ) @@ -246,14 +246,15 @@ def _store_call_info( @property def nav_subtitle(self): - cache_calls = len(self.calls) + stats = self.get_stats() + cache_calls = len(stats["calls"]) return ( ngettext( "%(cache_calls)d call in %(time).2fms", "%(cache_calls)d calls in %(time).2fms", cache_calls, ) - % {"cache_calls": cache_calls, "time": self.total_time} + % {"cache_calls": cache_calls, "time": stats["total_time"]} ) @property diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index 541c59136..3e328190d 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -12,6 +12,7 @@ from debug_toolbar.panels import Panel from debug_toolbar.panels.history import views from debug_toolbar.panels.history.forms import HistoryStoreForm +from debug_toolbar.store import get_store class HistoryPanel(Panel): @@ -46,9 +47,9 @@ def nav_subtitle(self): def generate_stats(self, request, response): try: if request.method == "GET": - data = request.GET.copy() + data = dict(request.GET.copy()) else: - data = request.POST.copy() + data = dict(request.POST.copy()) # GraphQL tends to not be populated in POST. If the request seems # empty, check if it's a JSON request. if ( @@ -80,23 +81,25 @@ def content(self): Fetch every store for the toolbar and include it in the template. """ - stores = OrderedDict() - for id, toolbar in reversed(self.toolbar._store.items()): - stores[id] = { - "toolbar": toolbar, - "form": SignedDataForm( - initial=HistoryStoreForm(initial={"store_id": id}).initial - ), - } + histories = OrderedDict() + for id in reversed(get_store().ids()): + stats = self.deserialize_stats(get_store().panel(id, self.panel_id)) + if stats: + histories[id] = { + "stats": stats, + "form": SignedDataForm( + initial=HistoryStoreForm(initial={"store_id": str(id)}).initial + ), + } return render_to_string( self.template, { "current_store_id": self.toolbar.store_id, - "stores": stores, + "histories": histories, "refresh_form": SignedDataForm( initial=HistoryStoreForm( - initial={"store_id": self.toolbar.store_id} + initial={"store_id": str(self.toolbar.store_id)} ).initial ), }, diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py index 10b4dcc1a..06b1a46b9 100644 --- a/debug_toolbar/panels/history/views.py +++ b/debug_toolbar/panels/history/views.py @@ -4,7 +4,8 @@ from debug_toolbar.decorators import require_show_toolbar, signed_data_view from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels.history.forms import HistoryStoreForm -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import get_store +from debug_toolbar.toolbar import stats_only_toolbar @require_show_toolbar @@ -15,7 +16,8 @@ def history_sidebar(request, verified_data): if form.is_valid(): store_id = form.cleaned_data["store_id"] - toolbar = DebugToolbar.fetch(store_id) + toolbar = stats_only_toolbar(store_id) + context = {} if toolbar is None: # When the store_id has been popped already due to @@ -25,6 +27,7 @@ def history_sidebar(request, verified_data): if not panel.is_historical: continue panel_context = {"panel": panel} + context[panel.panel_id] = { "button": render_to_string( "debug_toolbar/includes/panel_button.html", panel_context @@ -45,8 +48,8 @@ def history_refresh(request, verified_data): if form.is_valid(): requests = [] - # Convert to list to handle mutations happenening in parallel - for id, toolbar in list(DebugToolbar._store.items())[::-1]: + for id in reversed(get_store().ids()): + toolbar = stats_only_toolbar(id) requests.append( { "id": id, @@ -54,8 +57,10 @@ def history_refresh(request, verified_data): "debug_toolbar/panels/history_tr.html", { "id": id, - "store_context": { - "toolbar": toolbar, + "history": { + "stats": toolbar.get_panel_by_id( + "HistoryPanel" + ).get_stats(), "form": SignedDataForm( initial=HistoryStoreForm( initial={"store_id": id} diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index fdd5ed06e..21907effb 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -51,6 +51,7 @@ def __init__( self.id = id self.parent_ids = parent_ids self.hsv = hsv + self.has_subfuncs = False def parent_classes(self): return self.parent_classes @@ -141,6 +142,20 @@ def cumtime_per_call(self): def indent(self): return 16 * self.depth + def as_context(self): + return { + "id": self.id, + "parent_ids": self.parent_ids, + "func_std_string": self.func_std_string(), + "has_subfuncs": self.has_subfuncs, + "cumtime": self.cumtime(), + "cumtime_per_call": self.cumtime_per_call(), + "tottime": self.tottime(), + "tottime_per_call": self.tottime_per_call(), + "count": self.count(), + "indent": self.indent(), + } + class ProfilingPanel(Panel): """ @@ -157,7 +172,6 @@ def process_request(self, request): def add_node(self, func_list, func, max_depth, cum_time=0.1): func_list.append(func) - func.has_subfuncs = False if func.depth < max_depth: for subfunc in func.subfuncs(): if subfunc.stats[3] >= cum_time: @@ -183,4 +197,4 @@ def generate_stats(self, request, response): dt_settings.get_config()["PROFILER_MAX_DEPTH"], root.stats[3] / 8, ) - self.record_stats({"func_list": func_list}) + self.record_stats({"func_list": [func.as_context() for func in func_list]}) diff --git a/debug_toolbar/panels/request.py b/debug_toolbar/panels/request.py index 5255624b2..348f435ca 100644 --- a/debug_toolbar/panels/request.py +++ b/debug_toolbar/panels/request.py @@ -24,13 +24,11 @@ def nav_subtitle(self): return view_func.rsplit(".", 1)[-1] def generate_stats(self, request, response): - self.record_stats( - { - "get": get_sorted_request_variable(request.GET), - "post": get_sorted_request_variable(request.POST), - "cookies": get_sorted_request_variable(request.COOKIES), - } - ) + stats = { + "get": get_sorted_request_variable(request.GET), + "post": get_sorted_request_variable(request.POST), + "cookies": get_sorted_request_variable(request.COOKIES), + } view_info = { "view_func": _(""), @@ -56,10 +54,10 @@ def generate_stats(self, request, response): except Http404: pass - self.record_stats(view_info) + stats.update(view_info) if hasattr(request, "session"): - self.record_stats( + stats.update( { "session": [ (k, request.session.get(k)) @@ -67,3 +65,4 @@ def generate_stats(self, request, response): ] } ) + self.record_stats(stats) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index f8b92a5bd..37d0a85fd 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -110,18 +110,20 @@ def record(self, alias, **kwargs): @property def nav_subtitle(self): + stats = self.get_stats() + num_queries = len(stats["queries"]) return ngettext( "%(query_count)d query in %(sql_time).2fms", "%(query_count)d queries in %(sql_time).2fms", - self._num_queries, + num_queries, ) % { - "query_count": self._num_queries, - "sql_time": self._sql_time, + "query_count": num_queries, + "sql_time": stats["sql_time"], } @property def title(self): - count = len(self._databases) + count = len(self.get_stats()["databases"]) return ( ngettext( "SQL queries from %(count)d connection", @@ -144,10 +146,14 @@ def get_urls(cls): def enable_instrumentation(self): # This is thread-safe because database connections are thread-local. for connection in connections.all(): + if connection.alias == "debug_toolbar": + continue wrap_cursor(connection, self) def disable_instrumentation(self): for connection in connections.all(): + if connection.alias == "debug_toolbar": + continue unwrap_cursor(connection) def generate_stats(self, request, response): diff --git a/debug_toolbar/panels/staticfiles.py b/debug_toolbar/panels/staticfiles.py index d90b6501a..243569ece 100644 --- a/debug_toolbar/panels/staticfiles.py +++ b/debug_toolbar/panels/staticfiles.py @@ -82,9 +82,10 @@ class StaticFilesPanel(panels.Panel): @property def title(self): + stats = self.get_stats() return _("Static files (%(num_found)s found, %(num_used)s used)") % { - "num_found": self.num_found, - "num_used": self.num_used, + "num_found": stats["num_found"], + "num_used": stats["num_used"], } def __init__(self, *args, **kwargs): @@ -107,7 +108,7 @@ def num_used(self): @property def nav_subtitle(self): - num_used = self.num_used + num_used = self.get_stats()["num_used"] return ngettext( "%(num_used)s file used", "%(num_used)s files used", num_used ) % {"num_used": num_used} diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py index 8ff06e27d..efd799678 100644 --- a/debug_toolbar/panels/templates/panel.py +++ b/debug_toolbar/panels/templates/panel.py @@ -171,20 +171,24 @@ def disable_instrumentation(self): def generate_stats(self, request, response): template_context = [] for template_data in self.templates: - info = {} # Clean up some info about templates template = template_data["template"] if hasattr(template, "origin") and template.origin and template.origin.name: - template.origin_name = template.origin.name - template.origin_hash = signing.dumps(template.origin.name) + origin_name = template.origin.name + origin_hash = signing.dumps(template.origin.name) else: - template.origin_name = _("No origin") - template.origin_hash = "" - info["template"] = template - # Clean up context for better readability - if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]: - context_list = template_data.get("context", []) - info["context"] = "\n".join(context_list) + origin_name = _("No origin") + origin_hash = "" + info = { + "template": { + "origin_name": origin_name, + "origin_hash": origin_hash, + "name": template.name, + }, + "context": "\n".join(template_data.get("context", [])) + if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"] + else "", + } template_context.append(info) # Fetch context_processors/template_dirs from any template diff --git a/debug_toolbar/panels/timer.py b/debug_toolbar/panels/timer.py index 801c9c6fd..dff323c32 100644 --- a/debug_toolbar/panels/timer.py +++ b/debug_toolbar/panels/timer.py @@ -19,11 +19,9 @@ class TimerPanel(Panel): def nav_subtitle(self): stats = self.get_stats() - if hasattr(self, "_start_rusage"): - utime = self._end_rusage.ru_utime - self._start_rusage.ru_utime - stime = self._end_rusage.ru_stime - self._start_rusage.ru_stime + if stats.get("utime"): return _("CPU: %(cum)0.2fms (%(total)0.2fms)") % { - "cum": (utime + stime) * 1000.0, + "cum": stats["utime"], "total": stats["total_time"], } elif "total_time" in stats: diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index d8e6868a3..0f21689f0 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -21,6 +21,7 @@ "ROOT_TAG_EXTRA_ATTRS": "", "SHOW_COLLAPSED": False, "SHOW_TOOLBAR_CALLBACK": "debug_toolbar.middleware.show_toolbar", + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.MemoryStore", # Panel options "EXTRA_SIGNALS": [], "ENABLE_STACKTRACES": True, diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py new file mode 100644 index 000000000..a14059c4e --- /dev/null +++ b/debug_toolbar/store.py @@ -0,0 +1,90 @@ +import json +from collections import defaultdict + +from django.core.serializers.json import DjangoJSONEncoder +from django.utils.module_loading import import_string + +from debug_toolbar import settings as dt_settings + + +class DebugToolbarJSONEncoder(DjangoJSONEncoder): + def default(self, o): + try: + return super().default(o) + except TypeError: + return str(o) + + +def serialize(data): + return json.dumps(data, cls=DebugToolbarJSONEncoder) + + +def deserialize(data): + return json.loads(data) + + +# Record stats in serialized fashion. +# Remove use of fetching the toolbar as a whole from the store. + + +class BaseStore: + config = dt_settings.get_config().copy() + + @classmethod + def ids(cls): + raise NotImplementedError + + @classmethod + def exists(cls, store_id): + raise NotImplementedError + + @classmethod + def set(cls, store_id): + raise NotImplementedError + + @classmethod + def delete(cls, store_id): + raise NotImplementedError + + +class MemoryStore(BaseStore): + _ids = list() + _stats = defaultdict(dict) + + @classmethod + def ids(cls): + return cls._ids + + @classmethod + def exists(cls, store_id): + return store_id in cls._ids + + @classmethod + def set(cls, store_id): + if store_id not in cls._ids: + cls._ids.append(store_id) + for _ in range(len(cls._ids) - cls.config["RESULTS_CACHE_SIZE"]): + cls.delete(cls._ids[0]) + + @classmethod + def delete(cls, store_id): + if store_id in cls._stats: + del cls._stats[store_id] + if store_id in cls._ids: + cls._ids.remove(store_id) + + @classmethod + def save_panel(cls, store_id, panel_id, stats=None): + cls._stats[store_id][panel_id] = serialize(stats) + + @classmethod + def panel(cls, store_id, panel_id): + try: + data = cls._stats[store_id][panel_id] + except KeyError: + data = None + return {} if data is None else deserialize(data) + + +def get_store(): + return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/debug_toolbar/templates/debug_toolbar/panels/cache.html b/debug_toolbar/templates/debug_toolbar/panels/cache.html index 0e1ec2a4c..2390fff5a 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/cache.html +++ b/debug_toolbar/templates/debug_toolbar/panels/cache.html @@ -61,7 +61,8 @@

{% trans "Calls" %}

- 
{{ call.trace }}
+ {# The trace property is escaped when serialized into the store #} + 
{{ call.trace|safe }}
{% endfor %} diff --git a/debug_toolbar/templates/debug_toolbar/panels/history.html b/debug_toolbar/templates/debug_toolbar/panels/history.html index 84c6cb5bd..af95123e8 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/history.html +++ b/debug_toolbar/templates/debug_toolbar/panels/history.html @@ -15,7 +15,7 @@ - {% for id, store_context in stores.items %} + {% for id, history in histories.items %} {% include "debug_toolbar/panels/history_tr.html" %} {% endfor %} diff --git a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html index 31793472a..9029cf522 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html +++ b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html @@ -1,13 +1,13 @@ {% load i18n %} - {{ store_context.toolbar.stats.HistoryPanel.time|escape }} + {{ history.stats.time|escape }} -

{{ store_context.toolbar.stats.HistoryPanel.request_method|escape }}

+

{{ history.stats.request_method|escape }}

-

{{ store_context.toolbar.stats.HistoryPanel.request_url|truncatechars:100|escape }}

+

{{ history.stats.request_url|truncatechars:100|escape }}

@@ -24,7 +24,7 @@ - {% for key, value in store_context.toolbar.stats.HistoryPanel.data.items %} + {% for key, value in history.stats.data.items %} {{ key|pprint }} {{ value|pprint }} @@ -43,7 +43,7 @@
- {{ store_context.form }} + {{ history.form }}
diff --git a/debug_toolbar/templates/debug_toolbar/panels/profiling.html b/debug_toolbar/templates/debug_toolbar/panels/profiling.html index 837698889..86fe85770 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/profiling.html +++ b/debug_toolbar/templates/debug_toolbar/panels/profiling.html @@ -20,7 +20,8 @@ {% else %} {% endif %} - {{ call.func_std_string }} + {# The func_std_string property is escaped when serialized into the store #} + {{ call.func_std_string|safe }} {{ call.cumtime|floatformat:3 }} diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html index 6080e9f19..13da1d08d 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/sql.html +++ b/debug_toolbar/templates/debug_toolbar/panels/sql.html @@ -77,7 +77,8 @@ {% if query.params %} {% if query.is_select %}
- {{ query.form }} + {# The form is rendered when serialized into storage #} + {{ query.form|safe }} {% if query.vendor == 'mysql' %} @@ -100,7 +101,8 @@

{% trans "Transaction status:" %} {{ query.trans_status }}

{% endif %} {% if query.stacktrace %} - 
{{ query.stacktrace }}
+ {# The stacktrace property is a rendered template. It is escaped when serialized into the store #} + 
{{ query.stacktrace|safe }}
{% endif %} {% if query.template_info %} diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index cb886c407..56483674a 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -14,10 +14,12 @@ from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings +from debug_toolbar.store import get_store class DebugToolbar: - def __init__(self, request, get_response): + def __init__(self, request, get_response, store_id=None): + self.store_id = store_id or uuid.uuid4().hex self.request = request self.config = dt_settings.get_config().copy() panels = [] @@ -33,7 +35,6 @@ def __init__(self, request, get_response): self._panels[panel.panel_id] = panel self.stats = {} self.server_timing_stats = {} - self.store_id = None # Manage panels @@ -63,8 +64,7 @@ def render_toolbar(self): """ Renders the overall Toolbar with panels inside. """ - if not self.should_render_panels(): - self.store() + self.store() try: context = {"toolbar": self} return render_to_string("debug_toolbar/base.html", context) @@ -88,22 +88,8 @@ def should_render_panels(self): render_panels = self.request.META["wsgi.multiprocess"] return render_panels - # Handle storing toolbars in memory and fetching them later on - - _store = OrderedDict() - def store(self): - # Store already exists. - if self.store_id: - return - self.store_id = uuid.uuid4().hex - self._store[self.store_id] = self - for _ in range(self.config["RESULTS_CACHE_SIZE"], len(self._store)): - self._store.popitem(last=False) - - @classmethod - def fetch(cls, store_id): - return cls._store.get(store_id) + get_store().set(self.store_id) # Manually implement class-level caching of panel classes and url patterns # because it's more obvious than going through an abstraction. @@ -154,5 +140,9 @@ def is_toolbar_request(cls, request): return resolver_match.namespaces and resolver_match.namespaces[-1] == app_name +def stats_only_toolbar(store_id): + return DebugToolbar(request=None, get_response=lambda r: r, store_id=store_id) + + app_name = "djdt" urlpatterns = DebugToolbar.get_urls() diff --git a/debug_toolbar/views.py b/debug_toolbar/views.py index 1d319027d..c05b7305c 100644 --- a/debug_toolbar/views.py +++ b/debug_toolbar/views.py @@ -3,14 +3,15 @@ from django.utils.translation import gettext as _ from debug_toolbar.decorators import require_show_toolbar -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import get_store +from debug_toolbar.toolbar import stats_only_toolbar @require_show_toolbar def render_panel(request): """Render the contents of a panel""" - toolbar = DebugToolbar.fetch(request.GET["store_id"]) - if toolbar is None: + store_id = request.GET["store_id"] + if not get_store().exists(store_id): content = _( "Data for this panel isn't available anymore. " "Please reload the page and retry." @@ -18,6 +19,7 @@ def render_panel(request): content = "

%s

" % escape(content) scripts = [] else: + toolbar = stats_only_toolbar(store_id) panel = toolbar.get_panel_by_id(request.GET["panel_id"]) content = panel.content scripts = panel.scripts diff --git a/tests/base.py b/tests/base.py index c09828b4f..ca3e5f1d8 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,12 +2,14 @@ from django.http import HttpResponse from django.test import RequestFactory, TestCase +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar rf = RequestFactory() class BaseTestCase(TestCase): + databases = {"default", "debug_toolbar"} panel_id = None def setUp(self): @@ -48,10 +50,12 @@ def assertValidHTML(self, content, msg=None): class IntegrationTestCase(TestCase): """Base TestCase for tests involving clients making requests.""" + databases = {"default", "debug_toolbar"} + def setUp(self): # The HistoryPanel keeps track of previous stores in memory. # This bleeds into other tests and violates their idempotency. # Clear the store before each test. - for key in list(DebugToolbar._store.keys()): - del DebugToolbar._store[key] + for key in list(get_store().ids()): + get_store().delete(key) super().setUp() diff --git a/tests/commands/test_debugsqlshell.py b/tests/commands/test_debugsqlshell.py index 9520d0dd8..d5e8c03ae 100644 --- a/tests/commands/test_debugsqlshell.py +++ b/tests/commands/test_debugsqlshell.py @@ -16,6 +16,8 @@ @override_settings(DEBUG=True) class DebugSQLShellTestCase(TestCase): + databases = {"default", "debug_toolbar"} + def setUp(self): self.original_wrapper = base_module.CursorDebugWrapper # Since debugsqlshell monkey-patches django.db.backends.utils, we can diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py index 49e3bd0fa..165a3ec8a 100644 --- a/tests/panels/test_history.py +++ b/tests/panels/test_history.py @@ -4,7 +4,8 @@ from django.urls import resolve, reverse from debug_toolbar.forms import SignedDataForm -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import get_store +from debug_toolbar.toolbar import stats_only_toolbar from ..base import BaseTestCase, IntegrationTestCase @@ -25,7 +26,7 @@ def test_post(self): response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) data = self.panel.get_stats()["data"] - self.assertEqual(data["foo"], "bar") + self.assertEqual(data["foo"], ["bar"]) def test_post_json(self): for data, expected_stats_data in ( @@ -83,35 +84,53 @@ class HistoryViewsTestCase(IntegrationTestCase): def test_history_panel_integration_content(self): """Verify the history panel's content renders properly..""" - self.assertEqual(len(DebugToolbar._store), 0) + store = get_store() + self.assertEqual(len(store.ids()), 0) data = {"foo": "bar"} self.client.get("/json_view/", data, content_type="application/json") # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(DebugToolbar._store), 1) - toolbar = list(DebugToolbar._store.values())[0] + self.assertEqual(len(store.ids()), 1) + toolbar = stats_only_toolbar(store.ids()[0]) content = toolbar.get_panel_by_id("HistoryPanel").content self.assertIn("bar", content) def test_history_sidebar_invalid(self): + store = get_store() response = self.client.get(reverse("djdt:history_sidebar")) self.assertEqual(response.status_code, 400) - data = {"signed": SignedDataForm.sign({"store_id": "foo"}) + "invalid"} + self.client.get("/json_view/") + store_id = store.ids()[0] + data = {"signed": SignedDataForm.sign({"store_id": store_id}) + "invalid"} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 400) - def test_history_sidebar(self): - """Validate the history sidebar view.""" + def test_history_sidebar_hash(self): + """Validate the hashing mechanism.""" + store = get_store() self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] + store_id = store.ids()[0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) self.assertEqual( - set(response.json()), - self.PANEL_KEYS, + list(response.json().keys()), + [ + "VersionsPanel", + "TimerPanel", + "SettingsPanel", + "HeadersPanel", + "RequestPanel", + "SQLPanel", + "StaticFilesPanel", + "TemplatesPanel", + "CachePanel", + "SignalsPanel", + "LoggingPanel", + "ProfilingPanel", + ], ) @override_settings( @@ -119,8 +138,9 @@ def test_history_sidebar(self): ) def test_history_sidebar_expired_store_id(self): """Validate the history sidebar view.""" + store = get_store() self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] + store_id = list(store.ids())[0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) @@ -130,14 +150,18 @@ def test_history_sidebar_expired_store_id(self): ) self.client.get("/json_view/") - # Querying old store_id should return in empty response + # Querying previous store_id should still work data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) - self.assertEqual(response.json(), {}) + self.assertEqual( + set(response.json()), + self.PANEL_KEYS, + ) # Querying with latest store_id - latest_store_id = list(DebugToolbar._store)[0] + latest_store_id = store.ids()[-1] + self.assertNotEqual(latest_store_id, store_id) data = {"signed": SignedDataForm.sign({"store_id": latest_store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) @@ -157,15 +181,15 @@ def test_history_refresh_invalid_signature(self): def test_history_refresh(self): """Verify refresh history response has request variables.""" - data = {"foo": "bar"} - self.client.get("/json_view/", data, content_type="application/json") + store = get_store() + self.client.get("/json_view/", {"foo": "bar"}, content_type="application/json") data = {"signed": SignedDataForm.sign({"store_id": "foo"})} response = self.client.get(reverse("djdt:history_refresh"), data=data) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(len(data["requests"]), 1) - store_id = list(DebugToolbar._store)[0] + store_id = store.ids()[0] signature = SignedDataForm.sign({"store_id": store_id}) self.assertIn(html.escape(signature), data["requests"][0]["content"]) diff --git a/tests/settings.py b/tests/settings.py index 2a4b5e68c..5681bf948 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -92,6 +92,17 @@ "USER": "default_test", }, }, + "debug_toolbar": { + "ENGINE": "django.db.backends.%s" % os.getenv("DB_BACKEND"), + "NAME": "debug_toolbar_store", + "USER": os.getenv("DB_USER"), + "PASSWORD": os.getenv("DB_PASSWORD"), + "HOST": os.getenv("DB_HOST", ""), + "PORT": os.getenv("DB_PORT", ""), + "TEST": { + "USER": "default_test", + }, + }, } DEFAULT_AUTO_FIELD = "django.db.models.AutoField" diff --git a/tests/test_integration.py b/tests/test_integration.py index 3be1ef589..3402a8952 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,7 @@ import os import re import unittest +import uuid import django import html5lib @@ -16,7 +17,8 @@ from debug_toolbar.forms import SignedDataForm from debug_toolbar.middleware import DebugToolbarMiddleware, show_toolbar from debug_toolbar.panels import Panel -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import get_store +from debug_toolbar.toolbar import DebugToolbar, stats_only_toolbar from .base import BaseTestCase, IntegrationTestCase from .views import regular_view @@ -69,17 +71,17 @@ def test_url_resolving_positional(self): stats = self._resolve_stats("/resolving1/a/b/") self.assertEqual(stats["view_urlname"], "positional-resolving") self.assertEqual(stats["view_func"], "tests.views.resolving_view") - self.assertEqual(stats["view_args"], ("a", "b")) + self.assertEqual(stats["view_args"], ["a", "b"]) self.assertEqual(stats["view_kwargs"], {}) def test_url_resolving_named(self): stats = self._resolve_stats("/resolving2/a/b/") - self.assertEqual(stats["view_args"], ()) + self.assertEqual(stats["view_args"], []) self.assertEqual(stats["view_kwargs"], {"arg1": "a", "arg2": "b"}) def test_url_resolving_mixed(self): stats = self._resolve_stats("/resolving3/a/") - self.assertEqual(stats["view_args"], ("a",)) + self.assertEqual(stats["view_args"], ["a"]) self.assertEqual(stats["view_kwargs"], {"arg2": "default"}) def test_url_resolving_bad(self): @@ -189,7 +191,6 @@ def get_response(request): return HttpResponse() toolbar = DebugToolbar(rf.get("/"), get_response) - toolbar.store() url = "/__debug__/render_panel/" data = {"store_id": toolbar.store_id, "panel_id": "VersionsPanel"} @@ -207,18 +208,20 @@ def get_response(request): def test_middleware_render_toolbar_json(self): """Verify the toolbar is rendered and data is stored for a json request.""" - self.assertEqual(len(DebugToolbar._store), 0) + store = get_store() + self.assertEqual(len(store.ids()), 0) - data = {"foo": "bar"} + data = {"foo": "bar", "spam[]": ["eggs", "ham"]} response = self.client.get("/json_view/", data, content_type="application/json") self.assertEqual(response.status_code, 200) self.assertEqual(response.content.decode("utf-8"), '{"foo": "bar"}') # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(DebugToolbar._store), 1) - toolbar = list(DebugToolbar._store.values())[0] + self.assertEqual(len(store.ids()), 1) + toolbar = stats_only_toolbar(store.ids()[0]) + self.assertEqual( toolbar.get_panel_by_id("HistoryPanel").get_stats()["data"], - {"foo": ["bar"]}, + {"foo": ["bar"], "spam[]": ["eggs", "ham"]}, ) def test_template_source_checks_show_toolbar(self): @@ -442,6 +445,8 @@ def test_auth_login_view_without_redirect(self): ) @override_settings(DEBUG=True) class DebugToolbarLiveTestCase(StaticLiveServerTestCase): + databases = {"default", "debug_toolbar"} + @classmethod def setUpClass(cls): super().setUpClass() @@ -502,9 +507,10 @@ def test_basic_jinja(self): ) def test_rerender_on_history_switch(self): self.get("/regular_jinja/basic") + self.selenium.find_element_by_id("HistoryPanel") # Make a new request so the history panel has more than one option. self.get("/execute_sql/") - template_panel = self.selenium.find_element_by_id("HistoryPanel") + history_panel = self.selenium.find_element_by_id("HistoryPanel") # Record the current side panel of buttons for later comparison. previous_button_panel = self.selenium.find_element_by_id( "djDebugPanelList" @@ -513,7 +519,12 @@ def test_rerender_on_history_switch(self): # Click to show the history panel self.selenium.find_element_by_class_name("HistoryPanel").click() # Click to switch back to the jinja page view snapshot - list(template_panel.find_elements_by_css_selector("button"))[-1].click() + list(history_panel.find_elements_by_css_selector("button"))[-1].click() + + self.wait.until( + lambda selenium: self.selenium.find_element_by_id("djDebugPanelList").text + != previous_button_panel + ) current_button_panel = self.selenium.find_element_by_id("djDebugPanelList").text # Verify the button side panels have updated. @@ -521,11 +532,14 @@ def test_rerender_on_history_switch(self): self.assertNotIn("1 query", current_button_panel) self.assertIn("1 query", previous_button_panel) - @override_settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 0}) def test_expired_store(self): + store = get_store() self.get("/regular/basic/") version_panel = self.selenium.find_element_by_id("VersionsPanel") + for i in range(store.config["RESULTS_CACHE_SIZE"]): + store.set(uuid.uuid4().hex) + # Click to show the version panel self.selenium.find_element_by_class_name("VersionsPanel").click()