From c92d1bd615ecbf9cdf7fa09e697749ef2086033b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 12 Apr 2025 20:43:17 +0200 Subject: [PATCH 1/7] Fixes for decorating various function wrapper objs line_profiler/profiler_mixin.py[i] is_{static,bound,partial}method() is_partial() is_[cached_]property() New checker callables ByCountProfilerMixin.wrap_callable() Extended dispatch table ByCountProfilerMixin.wrap_classmethod() Fixed implementation; now returns a `classmethod` object ByCountProfilerMixin.wrap_{static,bound,partial}method() ByCountProfilerMixin.wrap_{partial,[cached_]property}() New methods --- line_profiler/profiler_mixin.py | 163 +++++++++++++++++++++++++++++-- line_profiler/profiler_mixin.pyi | 43 ++++++++ 2 files changed, 196 insertions(+), 10 deletions(-) diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index a12c7301..da05206f 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -1,5 +1,6 @@ import functools import inspect +import types is_coroutine = inspect.iscoroutinefunction @@ -10,6 +11,30 @@ def is_classmethod(f): return isinstance(f, classmethod) +def is_staticmethod(f): + return isinstance(f, staticmethod) + + +def is_boundmethod(f): + return isinstance(f, types.MethodType) + + +def is_partialmethod(f): + return isinstance(f, functools.partialmethod) + + +def is_partial(f): + return isinstance(f, functools.partial) + + +def is_property(f): + return isinstance(f, property) + + +def is_cached_property(f): + return isinstance(f, functools.cached_property) + + class ByCountProfilerMixin: """ Mixin class for profiler methods built around the @@ -25,6 +50,18 @@ def wrap_callable(self, func): """ if is_classmethod(func): wrapper = self.wrap_classmethod(func) + elif is_staticmethod(func): + wrapper = self.wrap_staticmethod(func) + elif is_boundmethod(func): + wrapper = self.wrap_boundmethod(func) + elif is_partialmethod(func): + wrapper = self.wrap_partialmethod(func) + elif is_partial(func): + wrapper = self.wrap_partial(func) + elif is_property(func): + wrapper = self.wrap_property(func) + elif is_cached_property(func): + wrapper = self.wrap_cached_property(func) elif is_coroutine(func): wrapper = self.wrap_coroutine(func) elif is_generator(func): @@ -33,19 +70,125 @@ def wrap_callable(self, func): wrapper = self.wrap_function(func) return wrapper - def wrap_classmethod(self, func): + def _wrap_callable_wrapper(self, wrapper, impl_attrs, *, + args=None, kwargs=None, name_attr=None): """ - Wrap a classmethod to profile it. + Create a profiled wrapper object around callables based on an + existing wrapper. + + Args: + wrapper (W): + Wrapper object around regular callables, like + `property`, `staticmethod`, `functools.partial`, etc. + impl_attrs (Sequence[str]): + Attribute names whence to retrieve the individual + callables to be wrapped and profiled, like `.fget`, + `.fset`, and `.fdel` for `property`; + the retrieved values are wrapped and passed as + positional arguments to the wrapper constructor. + args (Optional[str | Sequence[str]]): + Optional attribute name or names whence to retrieve + extra positional arguments to pass to the wrapper + constructor; + if a single name, the retrieved value is unpacked; + else, each name corresponds to one extra positional arg. + kwargs (Optional[str | Mapping[str, str]]): + Optional attribute name or name mapping whence to + retrieve extra keyword arguments to pass to the wrapper + constructor; + if a single name, the retrieved values is unpacked; + else, the attribute of `wrapper` at the mapping value is + used to populate the keyword arg at the mapping key. + name_attr (Optional[str]): + Optional attribute name whence to retrieve the name of + `wrapper` to be carried over in the new wrapper, like + `__name__` for `property` (Python 3.13+) and `attrname` + for `functools.cached_property`. + + Returns: + (W): new wrapper of the type of `wrapper` """ - @functools.wraps(func) - def wrapper(*args, **kwds): - self.enable_by_count() + # Wrap implementations + impls = [getattr(wrapper, attr) for attr in impl_attrs] + new_impls = [None if impl is None else self.wrap_callable(impl) + for impl in impls] + + # Get additional init args for the constructor + if args is None: + init_args = () + elif isinstance(args, str): + init_args = getattr(wrapper, args) + else: + init_args = [getattr(wrapper, attr) for attr in args] + if kwargs is None: + init_kwargs = {} + elif isinstance(kwargs, str): + init_kwargs = getattr(wrapper, kwargs) + else: + init_kwargs = {} + for name, attr in kwargs.items(): + try: + init_kwargs[name] = getattr(wrapper, attr) + except AttributeError: + pass + + new_wrapper = type(wrapper)(*new_impls, *init_args, **init_kwargs) + + # Metadata: descriptor name, instance dict + if name_attr: try: - result = func.__func__(func.__class__, *args, **kwds) - finally: - self.disable_by_count() - return result - return wrapper + setattr(new_wrapper, name_attr, getattr(wrapper, name_attr)) + except AttributeError: + pass + try: + old_vars = vars(wrapper) + new_vars = vars(new_wrapper) + except TypeError: # Object doesn't necessarily have a dict + pass + else: + for key, value in old_vars.items(): + new_vars.setdefault(key, value) + + return new_wrapper + + def _wrap_class_and_static_method(self, func): + """ + Wrap a class/static method to profile it. + """ + return self._wrap_callable_wrapper(func, ('__func__',)) + + wrap_classmethod = wrap_staticmethod = _wrap_class_and_static_method + + def wrap_boundmethod(self, func): + """ + Wrap a bound method to profile it. + """ + return self._wrap_callable_wrapper(func, ('__func__',), + args=('__self__',)) + + def _wrap_partial(self, func): + """ + Wrap a `functools.partial[method]` to profile it. + """ + return self._wrap_callable_wrapper(func, ('func',), + args='args', kwargs='keywords') + + wrap_partial = wrap_partialmethod = _wrap_partial + + def wrap_property(self, func): + """ + Wrap a property to profile it. + """ + return self._wrap_callable_wrapper(func, ('fget', 'fset', 'fdel'), + kwargs={'doc': '__doc__'}, + name_attr='__name__') + + def wrap_cached_property(self, func): + """ + Wrap a `functools.cached_property` to profile it. + """ + return self._wrap_callable_wrapper(func, ('func',), + name_attr='attrname') def wrap_coroutine(self, func): """ diff --git a/line_profiler/profiler_mixin.pyi b/line_profiler/profiler_mixin.pyi index 1f258480..b1c456bf 100644 --- a/line_profiler/profiler_mixin.pyi +++ b/line_profiler/profiler_mixin.pyi @@ -1,4 +1,5 @@ import inspect +import typing is_coroutine = inspect.iscoroutinefunction @@ -9,6 +10,30 @@ def is_classmethod(f) -> bool: ... +def is_staticmethod(f) -> bool: + ... + + +def is_boundmethod(f) -> bool: + ... + + +def is_partialmethod(f) -> bool: + ... + + +def is_partial(f) -> bool: + ... + + +def is_property(f) -> bool: + ... + + +def is_cached_property(f) -> bool: + ... + + class ByCountProfilerMixin: def wrap_callable(self, func): @@ -17,6 +42,24 @@ class ByCountProfilerMixin: def wrap_classmethod(self, func): ... + def wrap_staticmethod(self, func): + ... + + def wrap_boundmethod(self, func): + ... + + def wrap_partialmethod(self, func): + ... + + def wrap_partial(self, func): + ... + + def wrap_property(self, func): + ... + + def wrap_cached_property(self, func): + ... + def wrap_coroutine(self, func): ... From 089eb2fa2d825fe715f2277d1e3bc4c9d9b5aa45 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 13 Apr 2025 17:33:40 +0200 Subject: [PATCH 2/7] Fixed decoration; added tests line_profiler/line_profiler.py[i]::LineProfiler __call__() Added dispatch for calling `.add_function()` on the inner callables of callable-wrapper objects add_property() New helper method for calling `.add_function()` on the getter, setter, and deleter implementations of a property tests/test_line_profiler.py test_classmethod_decorator() Overhauled to test passing a real `classmethod` (instead of a bound method) to the profiler test_{static,bound,partial}method_decorator() test_{partial,[cached_]property}_decorator() New analogous tests for decorating/profiling other callable(-like) objects --- line_profiler/line_profiler.py | 24 ++- line_profiler/line_profiler.pyi | 3 + tests/test_line_profiler.py | 287 +++++++++++++++++++++++++++++++- 3 files changed, 303 insertions(+), 11 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 405a42e5..aa2867b1 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -19,7 +19,11 @@ 'The line_profiler._line_profiler c-extension is not importable. ' f'Has it been compiled? Underlying error is ex={ex!r}' ) -from .profiler_mixin import ByCountProfilerMixin +from .profiler_mixin import (ByCountProfilerMixin, + is_property, is_cached_property, + is_boundmethod, is_classmethod, is_staticmethod, + is_partial, is_partialmethod) + # NOTE: This needs to be in sync with ../kernprof.py and __init__.py __version__ = '4.3.0' @@ -54,9 +58,25 @@ def __call__(self, func): """ Decorate a function to start the profiler on function entry and stop it on function exit. """ - self.add_function(func) + if is_property(func): + self.add_property(func) + elif any(check(func) + for check in (is_boundmethod, + is_classmethod, is_staticmethod)): + self.add_function(func.__func__) + elif any(check(func) for check in (is_partial, is_partialmethod, + is_cached_property)): + self.add_function(func.func) + else: + self.add_function(func) return self.wrap_callable(func) + def add_property(self, func): + for impl in func.fget, func.fset, func.fdel: + if impl is None: + continue + self.add_function(impl) + def dump_stats(self, filename): """ Dump a representation of the data to a file as a pickled LineStats object from `get_stats()`. diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index b1d8bd10..9593e90f 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -12,6 +12,9 @@ def load_ipython_extension(ip) -> None: class LineProfiler(CLineProfiler, ByCountProfilerMixin): + def add_property(self, func) -> None: + ... + def dump_stats(self, filename) -> None: ... diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index 67ae6f03..09142e97 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -1,4 +1,9 @@ +import functools +import inspect +import io import sys +import textwrap +import types import pytest from line_profiler import LineProfiler @@ -18,11 +23,8 @@ def get_profiling_tool_name(): return sys.monitoring.get_tool(sys.monitoring.PROFILER_ID) -class C: - @classmethod - def c(self, value): - print(value) - return 0 +def strip(s): + return textwrap.dedent(s).strip('\n') def test_init(): @@ -101,13 +103,280 @@ def test_gen_decorator(): def test_classmethod_decorator(): + """ + Test for `LineProfiler.wrap_classmethod()`. + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator, as auto-profile normally does. + """ + profile = LineProfiler() + + class Object: + @profile + @classmethod + def foo(cls) -> str: + return cls.__name__ * 2 + + assert isinstance(inspect.getattr_static(Object, 'foo'), classmethod) + assert profile.enable_count == 0 + assert len(profile.functions) == 1 + assert Object.foo() == Object().foo() == 'ObjectObject' + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `Object.foo()` + assert output.endswith('- foo') + line, = (line for line in output.splitlines() if line.endswith('* 2')) + # Check that it has been run twice + assert int(line.split()[1]) == 2 + assert profile.enable_count == 0 + + +def test_staticmethod_decorator(): + """ + Test for `LineProfiler.wrap_staticmethod()`. + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator, as auto-profile normally does. + """ + profile = LineProfiler() + + class Object: + @profile + @staticmethod + def foo(x: int) -> int: + return x * 2 + + assert isinstance(inspect.getattr_static(Object, 'foo'), staticmethod) + assert profile.enable_count == 0 + assert len(profile.functions) == 1 + assert Object.foo(3) == Object().foo(3) == 6 + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `Object.foo()` + assert output.endswith('- foo') + line, = (line for line in output.splitlines() if line.endswith('* 2')) + # Check that it has been run twice + assert int(line.split()[1]) == 2 + assert profile.enable_count == 0 + + +def test_boundmethod_decorator(): + """ + Test for `LineProfiler.wrap_boundmethod()`. + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator, as auto-profile normally does. + """ + profile = LineProfiler() + + class Object: + def foo(self, x: int) -> int: + return id(self) * x + + obj = Object() + # Check that calls are aggregated + profiled_foo_1 = profile(obj.foo) + profiled_foo_2 = profile(obj.foo) + assert isinstance(profiled_foo_1, types.MethodType) + assert isinstance(profiled_foo_2, types.MethodType) + assert profile.enable_count == 0 + # XXX: should we try do remove duplicates? + assert profile.functions == [Object.foo, Object.foo] + assert (profiled_foo_1(2) + == profiled_foo_2(2) + == obj.foo(2) + == id(obj) * 2) + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `Object.foo()` + assert output.endswith('- foo') + line, = (line for line in output.splitlines() if line.endswith('* x')) + # Check that the wrapped methods has been run twice in total + assert int(line.split()[1]) == 2 + assert profile.enable_count == 0 + + +def test_partialmethod_decorator(): + """ + Test for `LineProfiler.wrap_partialmethod()` + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator in a function definition, as auto-profile normally does. + """ + profile = LineProfiler() + + class Object: + def foo(self, x: int) -> int: + return id(self) * x + + bar = profile(functools.partialmethod(foo, 1)) + + assert isinstance(inspect.getattr_static(Object, 'bar'), + functools.partialmethod) + obj = Object() + assert profile.enable_count == 0 + assert profile.functions == [Object.foo] + assert obj.foo(1) == obj.bar() == id(obj) + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `Object.foo()` (via `.bar()`) + assert output.endswith('- foo') + line, = (line for line in output.splitlines() if line.endswith('* x')) + # Check that the wrapped method has been run once + assert int(line.split()[1]) == 1 + assert profile.enable_count == 0 + + +def test_partial_decorator() -> None: + """ + Test for `LineProfiler.wrap_partial()`. + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator, as auto-profile normally does. + """ + profile = LineProfiler() + + def foo(x: int, y: int) -> int: + return x + y + + bar = functools.partial(foo, 2) + profiled_bar_1 = profile(bar) + profiled_bar_2 = profile(bar) + assert isinstance(profiled_bar_1, functools.partial) + assert isinstance(profiled_bar_2, functools.partial) + assert profile.enable_count == 0 + # XXX: should we try do remove duplicates? + assert profile.functions == [foo, foo] + assert (profiled_bar_1(3) + == profiled_bar_2(3) + == bar(3) + == foo(2, 3) + == 5) + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `foo()` + assert output.endswith('- foo') + line, = (line for line in output.splitlines() if line.endswith('x + y')) + # Check that the wrapped partials has been run twice in total + assert int(line.split()[1]) == 2 + assert profile.enable_count == 0 + + +def test_property_decorator(): + """ + Test for `LineProfiler.wrap_property()`. + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator, as auto-profile normally does. + """ profile = LineProfiler() - c_wrapped = profile(C.c) - assert c_wrapped.__name__ == 'c' + + with pytest.warns(UserWarning, + match='Adding a function with a __wrapped__ attribute'): + class Object: + def __init__(self, x: int) -> None: + self.x = x + + @profile + @property + def foo(self) -> int: + return self.x * 2 + + @profile + @foo.setter + def foo(self, foo) -> None: + self.x = foo // 2 + + assert isinstance(Object.foo, property) assert profile.enable_count == 0 - val = c_wrapped('test') + # XXX: should we try do remove duplicates? (getter added twice here) + # (that's also why there is the warning) + assert len(profile.functions) == 3 + obj = Object(3) + assert obj.foo == 6 # Use getter + obj.foo = 10 # Use setter + assert obj.x == 5 + assert obj.foo == 10 # Use getter + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `Object.foo` + assert output.endswith('- foo') + getter_line, = (line for line in output.splitlines() + if line.endswith('* 2')) + setter_line, = (line for line in output.splitlines() + if line.endswith('// 2')) + # Check that the getter has been run twice and the setter once + assert int(getter_line.split()[1]) == 2 + assert int(setter_line.split()[1]) == 1 + assert profile.enable_count == 0 + + +def test_cached_property_decorator(): + """ + Test for `LineProfiler.wrap_cached_property()` + + Notes + ----- + This is testing for an edge case; + for the best result, always use `@profile` as the innermost + decorator, as auto-profile normally does. + """ + profile = LineProfiler() + + class Object: + def __init__(self, x: int) -> None: + self.x = x + + @profile + @functools.cached_property + def foo(self) -> int: + return self.x * 2 + + assert isinstance(Object.foo, functools.cached_property) assert profile.enable_count == 0 - assert val == C.c('test') + assert len(profile.functions) == 1 + obj = Object(3) + assert obj.foo == 6 # Use getter + assert obj.foo == 6 # Getter not called because it's cached + with io.StringIO() as sio: + profile.print_stats(stream=sio, summarize=True) + output = strip(sio.getvalue()) + print(output) + # Check that we have profiled `Object.foo` + assert output.endswith('- foo') + line, = (line for line in output.splitlines() if line.endswith('* 2')) + # Check that the getter has been run once + assert int(line.split()[1]) == 1 assert profile.enable_count == 0 From e28d203e59d88ab2389eed2fd5c845a167677862 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 13 Apr 2025 20:23:58 +0200 Subject: [PATCH 3/7] Added handling for async generators line_profiler/profiler_mixin.py[i] is_async_generator() New checker function ByCountProfilerMixin.wrap_callable() Added dispatch to `.wrap_async_generator()` ByCountProfilerMixin.wrap_async_generator() New method for wrapping async generator functions ByCountProfilerMixin.wrap_generator() Simplified implementation tests/test_line_profiler.py test_function_decorator(), test_gen_decorator() - Added check for profiler timings before/after running the profiled functions - Added check that the profiled function is in `LineProfiler.functions` - (Only for `test_gen_decorator()`) added check that the wrapper is a generator function test_coroutine_decorator(), test_async_gen_decorator() New tests for wrapping coroutines and async generators --- line_profiler/profiler_mixin.py | 41 +++++++--- line_profiler/profiler_mixin.pyi | 4 + tests/test_line_profiler.py | 127 +++++++++++++++++++++++++++---- 3 files changed, 146 insertions(+), 26 deletions(-) diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index da05206f..5d694717 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -5,6 +5,7 @@ is_coroutine = inspect.iscoroutinefunction is_generator = inspect.isgeneratorfunction +is_async_generator = inspect.isasyncgenfunction def is_classmethod(f): @@ -62,6 +63,8 @@ def wrap_callable(self, func): wrapper = self.wrap_property(func) elif is_cached_property(func): wrapper = self.wrap_cached_property(func) + elif is_async_generator(func): + wrapper = self.wrap_async_generator(func) elif is_coroutine(func): wrapper = self.wrap_coroutine(func) elif is_generator(func): @@ -190,11 +193,31 @@ def wrap_cached_property(self, func): return self._wrap_callable_wrapper(func, ('func',), name_attr='attrname') + def wrap_async_generator(self, func): + """ + Wrap an async generator to profile it. + """ + @functools.wraps(func) + async def wrapper(*args, **kwds): + g = func(*args, **kwds) + # Async generators are started by `.asend(None)` + input_ = None + while True: + self.enable_by_count() + try: + item = (await g.asend(input_)) + except StopAsyncIteration: + return + finally: + self.disable_by_count() + input_ = (yield item) + + return wrapper + def wrap_coroutine(self, func): """ Wrap a Python 3.5 coroutine to profile it. """ - @functools.wraps(func) async def wrapper(*args, **kwds): self.enable_by_count() @@ -207,21 +230,14 @@ async def wrapper(*args, **kwds): return wrapper def wrap_generator(self, func): - """ Wrap a generator to profile it. + """ + Wrap a generator to profile it. """ @functools.wraps(func) def wrapper(*args, **kwds): g = func(*args, **kwds) - # The first iterate will not be a .send() - self.enable_by_count() - try: - item = next(g) - except StopIteration: - return - finally: - self.disable_by_count() - input_ = (yield item) - # But any following one might be. + # Generators are started by `.send(None)` + input_ = None while True: self.enable_by_count() try: @@ -231,6 +247,7 @@ def wrapper(*args, **kwds): finally: self.disable_by_count() input_ = (yield item) + return wrapper def wrap_function(self, func): diff --git a/line_profiler/profiler_mixin.pyi b/line_profiler/profiler_mixin.pyi index b1c456bf..0f751b06 100644 --- a/line_profiler/profiler_mixin.pyi +++ b/line_profiler/profiler_mixin.pyi @@ -4,6 +4,7 @@ import typing is_coroutine = inspect.iscoroutinefunction is_generator = inspect.isgeneratorfunction +is_async_generator = inspect.isasyncgenfunction def is_classmethod(f) -> bool: @@ -60,6 +61,9 @@ class ByCountProfilerMixin: def wrap_cached_property(self, func): ... + def wrap_async_generator(self, func): + ... + def wrap_coroutine(self, func): ... diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index 09142e97..8a6667be 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -1,3 +1,5 @@ +import asyncio +import contextlib import functools import inspect import io @@ -19,6 +21,18 @@ def g(x): yield y + 20 +async def ag(delay, start=()): + i = 0 + for x in start: + yield i + i += x + while True: + received = await asyncio.sleep(delay, (yield i)) + if received is None: + return + i += received + + def get_profiling_tool_name(): return sys.monitoring.get_tool(sys.monitoring.PROFILER_ID) @@ -27,6 +41,20 @@ def strip(s): return textwrap.dedent(s).strip('\n') +@contextlib.contextmanager +def check_timings(prof): + """ + Verify that the profiler starts without timing data and ends with + some. + """ + timings = prof.get_stats().timings + assert not any(timings.values()), ('Expected no timing entries, ' + f'got {timings!r}') + yield prof + timings = prof.get_stats().timings + assert any(timings.values()), f'Expected timing entries, got {timings!r}' + + def test_init(): lp = LineProfiler() assert lp.functions == [] @@ -75,31 +103,102 @@ def test_enable_disable(): def test_function_decorator(): + """ + Test for `LineProfiler.wrap_function()`. + """ profile = LineProfiler() f_wrapped = profile(f) + assert f in profile.functions assert f_wrapped.__name__ == 'f' - assert profile.enable_count == 0 - value = f_wrapped(10) - assert profile.enable_count == 0 - assert value == f(10) + with check_timings(profile): + assert profile.enable_count == 0 + value = f_wrapped(10) + assert profile.enable_count == 0 + assert value == f(10) def test_gen_decorator(): + """ + Test for `LineProfiler.wrap_generator()`. + """ profile = LineProfiler() g_wrapped = profile(g) + assert inspect.isgeneratorfunction(g_wrapped) + assert g in profile.functions assert g_wrapped.__name__ == 'g' - assert profile.enable_count == 0 - i = g_wrapped(10) - assert profile.enable_count == 0 - assert next(i) == 20 - assert profile.enable_count == 0 - assert i.send(30) == 50 - assert profile.enable_count == 0 - with pytest.raises(StopIteration): - next(i) - assert profile.enable_count == 0 + with check_timings(profile): + assert profile.enable_count == 0 + i = g_wrapped(10) + assert profile.enable_count == 0 + assert next(i) == 20 + assert profile.enable_count == 0 + assert i.send(30) == 50 + assert profile.enable_count == 0 + with pytest.raises(StopIteration): + next(i) + assert profile.enable_count == 0 + + +def test_coroutine_decorator(): + """ + Test for `LineProfiler.wrap_coroutine()`. + """ + async def coro(delay=.015625): + return (await asyncio.sleep(delay, 1)) + + profile = LineProfiler() + coro_wrapped = profile(coro) + assert inspect.iscoroutinefunction(coro) + assert coro in profile.functions + + with check_timings(profile): + assert profile.enable_count == 0 + assert asyncio.run(coro_wrapped()) == 1 + assert profile.enable_count == 0 + + +def test_async_gen_decorator(): + """ + Test for `LineProfiler.wrap_async_generator()`. + """ + delay = .015625 + + async def use_agen_complex(*args, delay=delay): + results = [] + agen = ag_wrapped(delay) + results.append(await agen.asend(None)) # Start the generator + for send in args: + with (pytest.raises(StopAsyncIteration) + if send is None else + contextlib.nullcontext()): + results.append(await agen.asend(send)) + if send is None: + break + return results + + async def use_agen_simple(*args, delay=delay): + results = [] + async for i in ag_wrapped(delay, args): + results.append(i) + return results + + profile = LineProfiler() + ag_wrapped = profile(ag) + assert inspect.isasyncgenfunction(ag_wrapped) + assert ag in profile.functions + + with check_timings(profile): + assert profile.enable_count == 0 + assert asyncio.run(use_agen_simple()) == [0] + assert profile.enable_count == 0 + assert asyncio.run(use_agen_simple(1, 2, 3)) == [0, 1, 3, 6] + assert profile.enable_count == 0 + assert asyncio.run(use_agen_complex(1, 2, 3)) == [0, 1, 3, 6] + assert profile.enable_count == 0 + assert asyncio.run(use_agen_complex(1, 2, 3, None, 4)) == [0, 1, 3, 6] + assert profile.enable_count == 0 def test_classmethod_decorator(): From 015dffe72da52e92f536390d615263554b9b4359 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 13 Apr 2025 20:36:15 +0200 Subject: [PATCH 4/7] Added CHANGELOG entry --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9a0c4253..0e242a51 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,7 @@ Changes * FIX: Fixed issue when calling ``kernprof`` with neither the ``-l`` nor ``-b`` flag; also refactored common methods to ``LineProfiler`` and ``ContextualProfile`` * FIX: Fixed auto-profiling of async function definitions #330 * ENH: Added CLI argument ``-m`` to ``kernprof`` for running a library module as a script; also made it possible for profiling targets to be supplied across multiple ``-p`` flags +* FIX: Fixed explicit profiling of class methods; added handling for profiling static, bound, and partial methods, ``functools.partial`` objects, (cached) properties, and async generator functions 4.2.0 ~~~~~ From 13e9a856749a3c552a3bdfb2a9df69f7da86519d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 13 Apr 2025 21:22:51 +0200 Subject: [PATCH 5/7] Minor `LineProfiler` refactoring line_profiler/line_profiler.py[i]::LineProfiler __call__() - Now deferring object registration to `.add_callable()` - Revised docstring add_callable() New wrapper method around the underlying `.add_function()` Cython method which expects functions, so that we can deal with callable-like wrappers like class methods, properties, etc. (Separated from `.__call__()`) add_property() Added docstring --- line_profiler/line_profiler.py | 19 ++++++++++++++++--- line_profiler/line_profiler.pyi | 3 +++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index aa2867b1..93a328b6 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -55,8 +55,18 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): """ def __call__(self, func): - """ Decorate a function to start the profiler on function entry and stop - it on function exit. + """ + Decorate a function, method, property, partial object etc. to + start the profiler on function entry and stop it on function + exit. + """ + self.add_callable(func) + return self.wrap_callable(func) + + def add_callable(self, func): + """ + Register a function, method, property, partial object, etc. with + the underlying Cython profiler. """ if is_property(func): self.add_property(func) @@ -69,9 +79,12 @@ def __call__(self, func): self.add_function(func.func) else: self.add_function(func) - return self.wrap_callable(func) def add_property(self, func): + """ + Register a `property`'s getter, setter, and deleter + implementations with the underlying Cython profiler. + """ for impl in func.fget, func.fset, func.fdel: if impl is None: continue diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 9593e90f..f74a909c 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -12,6 +12,9 @@ def load_ipython_extension(ip) -> None: class LineProfiler(CLineProfiler, ByCountProfilerMixin): + def add_callable(self, func) -> None: + ... + def add_property(self, func) -> None: ... From 5c8d54390bb3180a4698394e3ab5eb7f894da22a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 13 Apr 2025 23:23:40 +0200 Subject: [PATCH 6/7] Guard against wrapping an already-wrapped callable line_profiler/line_profiler.py::LineProfiler add_property() Rolled back into `add_callable()`, no longer a standalone method add_callable() - Now returning 1 if a function is added to the profiler, and 0 otherwise - Now checking against adding the wrapper callable around an already-added callable line_profiler/profiler_mixin.py::ByCountProfilerMixin wrap_{function,coroutine,[async_]generator}() - Now checking if `func` is a wrapper around an already-added callable, and returns the same wrapper without rewrapping if that is the case - Now marking wrappers so that they can be recognized and not rewrapped tests/test_line_profiler.py test_double_decoration() New test that double-wrapping doesn't erroneously profile the wrapper instead of the intended callable test_property_decorator() Updated because we are no longer double-adding the property getter --- line_profiler/line_profiler.py | 67 ++++++++++++++++++++++----------- line_profiler/profiler_mixin.py | 43 +++++++++++++++++---- tests/test_line_profiler.py | 50 ++++++++++++++++-------- 3 files changed, 115 insertions(+), 45 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 93a328b6..e2156d80 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -28,6 +28,8 @@ # NOTE: This needs to be in sync with ../kernprof.py and __init__.py __version__ = '4.3.0' +is_function = inspect.isfunction + def load_ipython_extension(ip): """ API for IPython to recognize this module as an IPython extension. @@ -36,6 +38,34 @@ def load_ipython_extension(ip): ip.register_magics(LineProfilerMagics) +def _get_underlying_functions(func): + """ + Get the underlying function objects of a callable or an adjacent + object. + + Returns: + funcs (list[Callable]) + """ + if any(check(func) + for check in (is_boundmethod, is_classmethod, is_staticmethod)): + return _get_underlying_functions(func.__func__) + if any(check(func) + for check in (is_partial, is_partialmethod, is_cached_property)): + return _get_underlying_functions(func.func) + if is_property(func): + result = [] + for impl in func.fget, func.fset, func.fdel: + if impl is not None: + result.extend(_get_underlying_functions(impl)) + return result + if not callable(func): + raise TypeError(f'func = {func!r}: ' + f'cannot get functions from {type(func)} objects') + if is_function(func): + return [func] + return [type(func).__call__] + + class LineProfiler(CLineProfiler, ByCountProfilerMixin): """ A profiler that records the execution times of individual lines. @@ -60,6 +90,9 @@ def __call__(self, func): start the profiler on function entry and stop it on function exit. """ + # Note: if `func` is a `types.FunctionType` which is already + # decorated by the profiler, the same object is returned; + # otherwise, wrapper objects are always returned. self.add_callable(func) return self.wrap_callable(func) @@ -67,28 +100,20 @@ def add_callable(self, func): """ Register a function, method, property, partial object, etc. with the underlying Cython profiler. - """ - if is_property(func): - self.add_property(func) - elif any(check(func) - for check in (is_boundmethod, - is_classmethod, is_staticmethod)): - self.add_function(func.__func__) - elif any(check(func) for check in (is_partial, is_partialmethod, - is_cached_property)): - self.add_function(func.func) - else: - self.add_function(func) - def add_property(self, func): - """ - Register a `property`'s getter, setter, and deleter - implementations with the underlying Cython profiler. + Returns: + 1 if any function is added to the profiler, 0 otherwise """ - for impl in func.fget, func.fset, func.fdel: - if impl is None: + guard = self._already_wrapped + + nadded = 0 + for impl in _get_underlying_functions(func): + if guard(impl): continue self.add_function(impl) + nadded += 1 + + return 1 if nadded else 0 def dump_stats(self, filename): """ Dump a representation of the data to a file as a pickled LineStats @@ -110,16 +135,16 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, def add_module(self, mod): """ Add all the functions in a module and its classes. """ - from inspect import isclass, isfunction + from inspect import isclass nfuncsadded = 0 for item in mod.__dict__.values(): if isclass(item): for k, v in item.__dict__.items(): - if isfunction(v): + if is_function(v): self.add_function(v) nfuncsadded += 1 - elif isfunction(item): + elif is_function(item): self.add_function(item) nfuncsadded += 1 diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index 5d694717..94653976 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -195,8 +195,12 @@ def wrap_cached_property(self, func): def wrap_async_generator(self, func): """ - Wrap an async generator to profile it. + Wrap an async generator function to profile it. """ + # Prevent double-wrap + if self._already_wrapped(func): + return func + @functools.wraps(func) async def wrapper(*args, **kwds): g = func(*args, **kwds) @@ -212,12 +216,16 @@ async def wrapper(*args, **kwds): self.disable_by_count() input_ = (yield item) - return wrapper + return self._mark_wrapped(wrapper) def wrap_coroutine(self, func): """ - Wrap a Python 3.5 coroutine to profile it. + Wrap a coroutine function to profile it. """ + # Prevent double-wrap + if self._already_wrapped(func): + return func + @functools.wraps(func) async def wrapper(*args, **kwds): self.enable_by_count() @@ -227,12 +235,16 @@ async def wrapper(*args, **kwds): self.disable_by_count() return result - return wrapper + return self._mark_wrapped(wrapper) def wrap_generator(self, func): """ - Wrap a generator to profile it. + Wrap a generator function to profile it. """ + # Prevent double-wrap + if self._already_wrapped(func): + return func + @functools.wraps(func) def wrapper(*args, **kwds): g = func(*args, **kwds) @@ -248,11 +260,16 @@ def wrapper(*args, **kwds): self.disable_by_count() input_ = (yield item) - return wrapper + return self._mark_wrapped(wrapper) def wrap_function(self, func): - """ Wrap a function to profile it. """ + Wrap a function to profile it. + """ + # Prevent double-wrap + if self._already_wrapped(func): + return func + @functools.wraps(func) def wrapper(*args, **kwds): self.enable_by_count() @@ -261,7 +278,15 @@ def wrapper(*args, **kwds): finally: self.disable_by_count() return result - return wrapper + + return self._mark_wrapped(wrapper) + + def _already_wrapped(self, func): + return getattr(func, self._profiler_wrapped_marker, None) == id(self) + + def _mark_wrapped(self, func): + setattr(func, self._profiler_wrapped_marker, id(self)) + return func def run(self, cmd): """ Profile a single executable statment in the main namespace. @@ -295,3 +320,5 @@ def __enter__(self): def __exit__(self, *_, **__): self.disable_by_count() + + _profiler_wrapped_marker = '__line_profiler_id__' diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index 8a6667be..b2a12028 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -102,6 +102,25 @@ def test_enable_disable(): assert lp.last_time == {} +def test_double_decoration(): + """ + Test that wrapping the same function twice does not result in + spurious profiling entries. + """ + profile = LineProfiler() + f_wrapped = profile(f) + f_double_wrapped = profile(f_wrapped) + assert f_double_wrapped is f_wrapped + + with check_timings(profile): + assert profile.enable_count == 0 + value = f_wrapped(10) + assert profile.enable_count == 0 + assert value == f(10) + + assert len(profile.get_stats().timings) == 1 + + def test_function_decorator(): """ Test for `LineProfiler.wrap_function()`. @@ -398,27 +417,26 @@ def test_property_decorator(): """ profile = LineProfiler() - with pytest.warns(UserWarning, - match='Adding a function with a __wrapped__ attribute'): - class Object: - def __init__(self, x: int) -> None: - self.x = x + class Object: + def __init__(self, x: int) -> None: + self.x = x + + @profile + @property + def foo(self) -> int: + return self.x * 2 - @profile - @property - def foo(self) -> int: - return self.x * 2 + # The profiler sees both the setter and the already-wrapped + # getter here, but it shouldn't re-wrap the getter - @profile - @foo.setter - def foo(self, foo) -> None: - self.x = foo // 2 + @profile + @foo.setter + def foo(self, foo) -> None: + self.x = foo // 2 assert isinstance(Object.foo, property) assert profile.enable_count == 0 - # XXX: should we try do remove duplicates? (getter added twice here) - # (that's also why there is the warning) - assert len(profile.functions) == 3 + assert len(profile.functions) == 2 obj = Object(3) assert obj.foo == 6 # Use getter obj.foo = 10 # Use setter From 7f25e704655088f399557da44fb21eb2de9e6282 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 13 Apr 2025 23:35:15 +0200 Subject: [PATCH 7/7] Scrubbed refactored-away method from stub file --- line_profiler/line_profiler.pyi | 3 --- 1 file changed, 3 deletions(-) diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index f74a909c..98811274 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -15,9 +15,6 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): def add_callable(self, func) -> None: ... - def add_property(self, func) -> None: - ... - def dump_stats(self, filename) -> None: ...