From bbf990daf76dce132ac2f66e782d463f54a21f7f 2022-12-03 20:27:08 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: 2022-12-03 20:27:08 Subject: [PATCH] Polish documentation, hide private functions --- diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 2f3b4f0..7dd585b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -50,7 +50,7 @@ Backward latex completion It is sometime challenging to know how to type a character, if you are using IPython, or any compatible frontend you can prepend backslash to the character -and press ```` to expand it to its latex form. +and press :kbd:`Tab` to expand it to its latex form. .. code:: @@ -59,7 +59,7 @@ and press ```` to expand it to its latex form. Both forward and backward completions can be deactivated by setting the -``Completer.backslash_combining_completions`` option to ``False``. +:any:`Completer.backslash_combining_completions` option to ``False``. Experimental @@ -95,7 +95,7 @@ having to execute any code: ... myvar[1].bi Tab completion will be able to infer that ``myvar[1]`` is a real number without -executing any code unlike the previously available ``IPCompleter.greedy`` +executing almost any code unlike the deprecated :any:`IPCompleter.greedy` option. Be sure to update :any:`jedi` to the latest stable version or to try the @@ -972,29 +972,38 @@ class Completer(Configurable): help="""Activate greedy completion. .. deprecated:: 8.8 - Use :any:`evaluation` and :any:`auto_close_dict_keys` instead. + Use :any:`Completer.evaluation` and :any:`Completer.auto_close_dict_keys` instead. - When enabled in IPython 8.8+ activates following settings for compatibility: - - ``evaluation = 'unsafe'`` - - ``auto_close_dict_keys = True`` + When enabled in IPython 8.8 or newer, changes configuration as follows: + + - ``Completer.evaluation = 'unsafe'`` + - ``Completer.auto_close_dict_keys = True`` """, ).tag(config=True) evaluation = Enum( ("forbidden", "minimal", "limited", "unsafe", "dangerous"), default_value="limited", - help="""Code evaluation under completion. + help="""Policy for code evaluation under completion. - Successive options allow to enable more eager evaluation for more accurate completion suggestions, - including for nested dictionaries, nested lists, or even results of function calls. Setting `unsafe` - or higher can lead to evaluation of arbitrary user code on TAB with potentially dangerous side effects. + Successive options allow to enable more eager evaluation for better + completion suggestions, including for nested dictionaries, nested lists, + or even results of function calls. + Setting ``unsafe`` or higher can lead to evaluation of arbitrary user + code on :kbd:`Tab` with potentially unwanted or dangerous side effects. Allowed values are: - - `forbidden`: no evaluation at all - - `minimal`: evaluation of literals and access to built-in namespaces; no item/attribute evaluation nor access to locals/globals - - `limited` (default): access to all namespaces, evaluation of hard-coded methods (``keys()``, ``__getattr__``, ``__getitems__``, etc) on allow-listed objects (e.g. ``dict``, ``list``, ``tuple``, ``pandas.Series``) - - `unsafe`: evaluation of all methods and function calls but not of syntax with side-effects like `del x`, - - `dangerous`: completely arbitrary evaluation + + - ``forbidden``: no evaluation of code is permitted, + - ``minimal``: evaluation of literals and access to built-in namespace; + no item/attribute evaluation nor access to locals/globals, + - ``limited``: access to all namespaces, evaluation of hard-coded methods + (for example: :any:`dict.keys`, :any:`object.__getattr__`, + :any:`object.__getitem__`) on allow-listed objects (for example: + :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``), + - ``unsafe``: evaluation of all methods and function calls but not of + syntax with side-effects like `del x`, + - ``dangerous``: completely arbitrary evaluation. """, ).tag(config=True) @@ -1019,7 +1028,15 @@ class Completer(Configurable): "unicode characters back to latex commands.").tag(config=True) auto_close_dict_keys = Bool( - False, help="""Enable auto-closing dictionary keys.""" + False, + help=""" + Enable auto-closing dictionary keys. + + When enabled string keys will be suffixed with a final quote + (matching the opening quote), tuple keys will also receive a + separating comma if needed, and keys which are final will + receive a closing bracket (``]``). + """, ).tag(config=True) def __init__(self, namespace=None, global_namespace=None, **kwargs): @@ -1157,8 +1174,8 @@ class Completer(Configurable): obj = guarded_eval( expr, EvaluationContext( - globals_=self.global_namespace, - locals_=self.namespace, + globals=self.global_namespace, + locals=self.namespace, evaluation=self.evaluation, ), ) @@ -1183,7 +1200,7 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -class DictKeyState(enum.Flag): +class _DictKeyState(enum.Flag): """Represent state of the key match in context of other possible matches. - given `d1 = {'a': 1}` completion on `d1['` will yield `{'a': END_OF_ITEM}` as there is no tuple. @@ -1199,6 +1216,7 @@ class DictKeyState(enum.Flag): def _parse_tokens(c): + """Parse tokens even if there is an error.""" tokens = [] token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__) while True: @@ -1257,7 +1275,7 @@ def match_dict_keys( prefix: str, delims: str, extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None, -) -> Tuple[str, int, Dict[str, DictKeyState]]: +) -> Tuple[str, int, Dict[str, _DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters @@ -1307,8 +1325,8 @@ def match_dict_keys( return True filtered_key_is_final: Dict[ - Union[str, bytes, int, float], DictKeyState - ] = defaultdict(lambda: DictKeyState.BASELINE) + Union[str, bytes, int, float], _DictKeyState + ] = defaultdict(lambda: _DictKeyState.BASELINE) for k in keys: # If at least one of the matches is not final, mark as undetermined. @@ -1319,9 +1337,9 @@ def match_dict_keys( if filter_prefix_tuple(k): key_fragment = k[prefix_tuple_size] filtered_key_is_final[key_fragment] |= ( - DictKeyState.END_OF_TUPLE + _DictKeyState.END_OF_TUPLE if len(k) == prefix_tuple_size + 1 - else DictKeyState.IN_TUPLE + else _DictKeyState.IN_TUPLE ) elif prefix_tuple_size > 0: # we are completing a tuple but this key is not a tuple, @@ -1329,7 +1347,7 @@ def match_dict_keys( pass else: if isinstance(k, text_serializable_types): - filtered_key_is_final[k] |= DictKeyState.END_OF_ITEM + filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM filtered_keys = filtered_key_is_final.keys() @@ -1367,7 +1385,7 @@ def match_dict_keys( token_start = token_match.start() token_prefix = token_match.group() - matched: Dict[str, DictKeyState] = {} + matched: Dict[str, _DictKeyState] = {} str_key: Union[str, bytes] @@ -2503,8 +2521,8 @@ class IPCompleter(Completer): tuple_prefix = guarded_eval( prior_tuple_keys, EvaluationContext( - globals_=self.global_namespace, - locals_=self.namespace, + globals=self.global_namespace, + locals=self.namespace, evaluation=self.evaluation, in_subscript=True, ), @@ -2569,7 +2587,7 @@ class IPCompleter(Completer): results = [] - end_of_tuple_or_item = DictKeyState.END_OF_TUPLE | DictKeyState.END_OF_ITEM + end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM for k, state_flag in matches.items(): result = leading + k @@ -2584,7 +2602,7 @@ class IPCompleter(Completer): if state_flag in end_of_tuple_or_item and can_close_bracket: result += "]" - if state_flag == DictKeyState.IN_TUPLE and can_close_tuple_item: + if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item: result += ", " results.append(result) return results diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index a510d38..637d329 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -17,6 +17,7 @@ from functools import cached_property from dataclasses import dataclass, field from IPython.utils.docs import GENERATING_DOCUMENTATION +from IPython.utils.decorators import undoc if TYPE_CHECKING or GENERATING_DOCUMENTATION: @@ -26,21 +27,25 @@ else: Protocol = object # requires Python >=3.8 +@undoc class HasGetItem(Protocol): def __getitem__(self, key) -> None: ... +@undoc class InstancesHaveGetItem(Protocol): def __call__(self, *args, **kwargs) -> HasGetItem: ... +@undoc class HasGetAttr(Protocol): def __getattr__(self, key) -> None: ... +@undoc class DoesNotHaveGetAttr(Protocol): pass @@ -49,7 +54,7 @@ class DoesNotHaveGetAttr(Protocol): MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] -def unbind_method(func: Callable) -> Union[Callable, None]: +def _unbind_method(func: Callable) -> Union[Callable, None]: """Get unbound method for given bound method. Returns None if cannot get unbound method.""" @@ -69,8 +74,11 @@ def unbind_method(func: Callable) -> Union[Callable, None]: return None +@undoc @dataclass class EvaluationPolicy: + """Definition of evaluation policy.""" + allow_locals_access: bool = False allow_globals_access: bool = False allow_item_access: bool = False @@ -92,12 +100,12 @@ class EvaluationPolicy: if func in self.allowed_calls: return True - owner_method = unbind_method(func) + owner_method = _unbind_method(func) if owner_method and owner_method in self.allowed_calls: return True -def has_original_dunder_external( +def _has_original_dunder_external( value, module_name, access_path, @@ -121,7 +129,7 @@ def has_original_dunder_external( return False -def has_original_dunder( +def _has_original_dunder( value, allowed_types, allowed_methods, allowed_external, method_name ): # note: Python ignores `__getattr__`/`__getitem__` on instances, @@ -141,12 +149,13 @@ def has_original_dunder( return True for module_name, *access_path in allowed_external: - if has_original_dunder_external(value, module_name, access_path, method_name): + if _has_original_dunder_external(value, module_name, access_path, method_name): return True return False +@undoc @dataclass class SelectivePolicy(EvaluationPolicy): allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) @@ -155,14 +164,14 @@ class SelectivePolicy(EvaluationPolicy): allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) def can_get_attr(self, value, attr): - has_original_attribute = has_original_dunder( + has_original_attribute = _has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattribute_methods, allowed_external=self.allowed_getattr_external, method_name="__getattribute__", ) - has_original_attr = has_original_dunder( + has_original_attr = _has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattr_methods, @@ -182,7 +191,7 @@ class SelectivePolicy(EvaluationPolicy): def can_get_item(self, value, item): """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" - return has_original_dunder( + return _has_original_dunder( value, allowed_types=self.allowed_getitem, allowed_methods=self._getitem_methods, @@ -211,34 +220,50 @@ class SelectivePolicy(EvaluationPolicy): } -class DummyNamedTuple(NamedTuple): +class _DummyNamedTuple(NamedTuple): pass class EvaluationContext(NamedTuple): - locals_: dict - globals_: dict + #: Local namespace + locals: dict + #: Global namespace + globals: dict + #: Evaluation policy identifier evaluation: Literal[ "forbidden", "minimal", "limited", "unsafe", "dangerous" ] = "forbidden" + #: Whether the evalution of code takes place inside of a subscript. + #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. in_subscript: bool = False -class IdentitySubscript: +class _IdentitySubscript: + """Returns the key itself when item is requested via subscript.""" + def __getitem__(self, key): return key -IDENTITY_SUBSCRIPT = IdentitySubscript() +IDENTITY_SUBSCRIPT = _IdentitySubscript() SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" -class GuardRejection(ValueError): +class GuardRejection(Exception): + """Exception raised when guard rejects evaluation attempt.""" + pass def guarded_eval(code: str, context: EvaluationContext): - locals_ = context.locals_ + """Evaluate provided code in the evaluation context. + + If evaluation policy given by context is set to ``forbidden`` + no evaluation will be performed; if it is set to ``dangerous`` + standard :func:`eval` will be used; finally, for any other, + policy :func:`eval_node` will be called on parsed AST. + """ + locals_ = context.locals if context.evaluation == "forbidden": raise GuardRejection("Forbidden mode") @@ -256,10 +281,10 @@ def guarded_eval(code: str, context: EvaluationContext): locals_ = locals_.copy() locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT code = SUBSCRIPT_MARKER + "[" + code + "]" - context = EvaluationContext(**{**context._asdict(), **{"locals_": locals_}}) + context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}}) if context.evaluation == "dangerous": - return eval(code, context.globals_, context.locals_) + return eval(code, context.globals, context.locals) expression = ast.parse(code, mode="eval") @@ -267,14 +292,12 @@ def guarded_eval(code: str, context: EvaluationContext): def eval_node(node: Union[ast.AST, None], context: EvaluationContext): - """ - Evaluate AST node in provided context. + """Evaluate AST node in provided context. - Applies evaluation restrictions defined in the context. + Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments. - Currently does not support evaluation of functions with keyword arguments. + Does not evaluate actions that always have side effects: - Does not evaluate actions which always have side effects: - class definitions (``class sth: ...``) - function definitions (``def sth: ...``) - variable assignments (``x = 1``) @@ -282,13 +305,15 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): - deletions (``del x``) Does not evaluate operations which do not return values: + - assertions (``assert x``) - pass (``pass``) - imports (``import x``) - - control flow - - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) - - loops (``for`` and `while``) - - exception handling + - control flow: + + - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) + - loops (``for`` and `while``) + - exception handling The purpose of this function is to guard against unwanted side-effects; it does not give guarantees on protection from malicious code execution. @@ -376,10 +401,10 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): f" not allowed in {context.evaluation} mode", ) if isinstance(node, ast.Name): - if policy.allow_locals_access and node.id in context.locals_: - return context.locals_[node.id] - if policy.allow_globals_access and node.id in context.globals_: - return context.globals_[node.id] + if policy.allow_locals_access and node.id in context.locals: + return context.locals[node.id] + if policy.allow_globals_access and node.id in context.globals: + return context.globals[node.id] if policy.allow_builtins_access and hasattr(builtins, node.id): # note: do not use __builtins__, it is implementation detail of Python return getattr(builtins, node.id) @@ -439,8 +464,8 @@ BUILTIN_GETITEM: Set[InstancesHaveGetItem] = { collections.UserDict, collections.UserList, collections.UserString, - DummyNamedTuple, - IdentitySubscript, + _DummyNamedTuple, + _IdentitySubscript, } @@ -537,3 +562,12 @@ EVALUATION_POLICIES = { allow_any_calls=True, ), } + + +__all__ = [ + "guarded_eval", + "eval_node", + "GuardRejection", + "EvaluationContext", + "_unbind_method", +] diff --git a/IPython/core/magics/config.py b/IPython/core/magics/config.py index 87fe3ee..9e1cb38 100644 --- a/IPython/core/magics/config.py +++ b/IPython/core/magics/config.py @@ -68,94 +68,22 @@ class ConfigMagics(Magics): To view what is configurable on a given class, just pass the class name:: - In [2]: %config IPCompleter - IPCompleter(Completer) options - ---------------------------- - IPCompleter.backslash_combining_completions= - Enable unicode completions, e.g. \\alpha . Includes completion of latex - commands, unicode names, and expanding unicode characters back to latex - commands. - Current: True - IPCompleter.debug= - Enable debug for the Completer. Mostly print extra information for - experimental jedi integration. + In [2]: %config LoggingMagics + LoggingMagics(Magics) options + --------------------------- + LoggingMagics.quiet= + Suppress output of log state when logging is enabled Current: False - IPCompleter.disable_matchers=... - List of matchers to disable. - The list should contain matcher identifiers (see - :any:`completion_matcher`). - Current: [] - IPCompleter.greedy= - Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. - Current: False - IPCompleter.jedi_compute_type_timeout= - Experimental: restrict time (in milliseconds) during which Jedi can compute types. - Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt - performance by preventing jedi to build its cache. - Current: 400 - IPCompleter.limit_to__all__= - DEPRECATED as of version 5.0. - Instruct the completer to use __all__ for the completion - Specifically, when completing on ``object.``. - When True: only those names in obj.__all__ will be included. - When False [default]: the __all__ attribute is ignored - Current: False - IPCompleter.merge_completions= - Whether to merge completion results into a single list - If False, only the completion results from the first non-empty - completer will be returned. - As of version 8.6.0, setting the value to ``False`` is an alias for: - ``IPCompleter.suppress_competing_matchers = True.``. - Current: True - IPCompleter.omit__names= - Instruct the completer to omit private method names - Specifically, when completing on ``object.``. - When 2 [default]: all names that start with '_' will be excluded. - When 1: all 'magic' names (``__foo__``) will be excluded. - When 0: nothing will be excluded. - Choices: any of [0, 1, 2] - Current: 2 - IPCompleter.profile_completions= - If True, emit profiling data for completion subsystem using cProfile. - Current: False - IPCompleter.profiler_output_dir= - Template for path at which to output profile data for completions. - Current: '.completion_profiles' - IPCompleter.suppress_competing_matchers= - Whether to suppress completions from other *Matchers*. - When set to ``None`` (default) the matchers will attempt to auto-detect - whether suppression of other matchers is desirable. For example, at the - beginning of a line followed by `%` we expect a magic completion to be the - only applicable option, and after ``my_dict['`` we usually expect a - completion with an existing dictionary key. - If you want to disable this heuristic and see completions from all matchers, - set ``IPCompleter.suppress_competing_matchers = False``. To disable the - heuristic for specific matchers provide a dictionary mapping: - ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': - False}``. - Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions - to the set of matchers with the highest priority; this is equivalent to - ``IPCompleter.merge_completions`` and can be beneficial for performance, but - will sometimes omit relevant candidates from matchers further down the - priority list. - Current: None - IPCompleter.use_jedi= - Experimental: Use Jedi to generate autocompletions. Default to True if jedi - is installed. - Current: True but the real use is in setting values:: - In [3]: %config IPCompleter.greedy = True + In [3]: %config LoggingMagics.quiet = True and these values are read from the user_ns if they are variables:: - In [4]: feeling_greedy=False + In [4]: feeling_quiet=False - In [5]: %config IPCompleter.greedy = feeling_greedy + In [5]: %config LoggingMagics.quiet = feeling_quiet """ from traitlets.config.loader import Config diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index b908f2a..9c98b7a 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -3,18 +3,18 @@ from IPython.core.guarded_eval import ( EvaluationContext, GuardRejection, guarded_eval, - unbind_method, + _unbind_method, ) from IPython.testing import decorators as dec import pytest def limited(**kwargs): - return EvaluationContext(locals_=kwargs, globals_={}, evaluation="limited") + return EvaluationContext(locals=kwargs, globals={}, evaluation="limited") def unsafe(**kwargs): - return EvaluationContext(locals_=kwargs, globals_={}, evaluation="unsafe") + return EvaluationContext(locals=kwargs, globals={}, evaluation="unsafe") @dec.skip_without("pandas") @@ -206,7 +206,7 @@ def test_access_builtins(): def test_subscript(): context = EvaluationContext( - locals_={}, globals_={}, evaluation="limited", in_subscript=True + locals={}, globals={}, evaluation="limited", in_subscript=True ) empty_slice = slice(None, None, None) assert guarded_eval("", context) == tuple() @@ -221,8 +221,8 @@ def test_unbind_method(): return "CUSTOM" x = X() - assert unbind_method(x.index) is X.index - assert unbind_method([].index) is list.index + assert _unbind_method(x.index) is X.index + assert _unbind_method([].index) is list.index def test_assumption_instance_attr_do_not_matter():