from typing import ( Any, Callable, Set, Tuple, NamedTuple, Type, Literal, Union, TYPE_CHECKING, ) import builtins import collections import sys import ast from functools import cached_property from dataclasses import dataclass, field from IPython.utils.docs import GENERATING_DOCUMENTATION from IPython.utils.decorators import undoc if TYPE_CHECKING or GENERATING_DOCUMENTATION: from typing_extensions import Protocol else: # do not require on runtime Protocol = object # requires Python >=3.8 @undoc class HasGetItem(Protocol): def __getitem__(self, key) -> None: ... @undoc class InstancesHaveGetItem(Protocol): def __call__(self, *args, **kwargs) -> HasGetItem: ... @undoc class HasGetAttr(Protocol): def __getattr__(self, key) -> None: ... @undoc class DoesNotHaveGetAttr(Protocol): pass # By default `__getattr__` is not explicitly implemented on most objects MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] def _unbind_method(func: Callable) -> Union[Callable, None]: """Get unbound method for given bound method. Returns None if cannot get unbound method.""" owner = getattr(func, "__self__", None) owner_class = type(owner) name = getattr(func, "__name__", None) instance_dict_overrides = getattr(owner, "__dict__", None) if ( owner is not None and name and ( not instance_dict_overrides or (instance_dict_overrides and name not in instance_dict_overrides) ) ): return getattr(owner_class, name) return None @undoc @dataclass class EvaluationPolicy: """Definition of evaluation policy.""" allow_locals_access: bool = False allow_globals_access: bool = False allow_item_access: bool = False allow_attr_access: bool = False allow_builtins_access: bool = False allow_any_calls: bool = False allowed_calls: Set[Callable] = field(default_factory=set) def can_get_item(self, value, item): return self.allow_item_access def can_get_attr(self, value, attr): return self.allow_attr_access def can_call(self, func): if self.allow_any_calls: return True if func in self.allowed_calls: return True owner_method = _unbind_method(func) if owner_method and owner_method in self.allowed_calls: return True def _has_original_dunder_external( value, module_name, access_path, method_name, ): try: if module_name not in sys.modules: return False member_type = sys.modules[module_name] for attr in access_path: member_type = getattr(member_type, attr) value_type = type(value) if type(value) == member_type: return True if isinstance(value, member_type): method = getattr(value_type, method_name, None) member_method = getattr(member_type, method_name, None) if member_method == method: return True except (AttributeError, KeyError): return False def _has_original_dunder( value, allowed_types, allowed_methods, allowed_external, method_name ): # note: Python ignores `__getattr__`/`__getitem__` on instances, # we only need to check at class level value_type = type(value) # strict type check passes → no need to check method if value_type in allowed_types: return True method = getattr(value_type, method_name, None) if not method: return None if method in allowed_methods: return True for module_name, *access_path in allowed_external: if _has_original_dunder_external(value, module_name, access_path, method_name): return True return False @undoc @dataclass class SelectivePolicy(EvaluationPolicy): allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set) allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set) allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) def can_get_attr(self, value, attr): has_original_attribute = _has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattribute_methods, allowed_external=self.allowed_getattr_external, method_name="__getattribute__", ) has_original_attr = _has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattr_methods, allowed_external=self.allowed_getattr_external, method_name="__getattr__", ) # Many objects do not have `__getattr__`, this is fine if has_original_attr is None and has_original_attribute: return True # Accept objects without modifications to `__getattr__` and `__getattribute__` return has_original_attr and has_original_attribute def get_attr(self, value, attr): if self.can_get_attr(value, attr): return getattr(value, attr) def can_get_item(self, value, item): """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" return _has_original_dunder( value, allowed_types=self.allowed_getitem, allowed_methods=self._getitem_methods, allowed_external=self.allowed_getitem_external, method_name="__getitem__", ) @cached_property def _getitem_methods(self) -> Set[Callable]: return self._safe_get_methods(self.allowed_getitem, "__getitem__") @cached_property def _getattr_methods(self) -> Set[Callable]: return self._safe_get_methods(self.allowed_getattr, "__getattr__") @cached_property def _getattribute_methods(self) -> Set[Callable]: return self._safe_get_methods(self.allowed_getattr, "__getattribute__") def _safe_get_methods(self, classes, name) -> Set[Callable]: return { method for class_ in classes for method in [getattr(class_, name, None)] if method } class _DummyNamedTuple(NamedTuple): pass class EvaluationContext(NamedTuple): #: Local namespace locals: dict #: Global namespace globals: dict #: Evaluation policy identifier evaluation: Literal[ "forbidden", "minimal", "limited", "unsafe", "dangerous" ] = "forbidden" #: Whether the evalution of code takes place inside of a subscript. #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. in_subscript: bool = False class _IdentitySubscript: """Returns the key itself when item is requested via subscript.""" def __getitem__(self, key): return key IDENTITY_SUBSCRIPT = _IdentitySubscript() SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" class GuardRejection(Exception): """Exception raised when guard rejects evaluation attempt.""" pass def guarded_eval(code: str, context: EvaluationContext): """Evaluate provided code in the evaluation context. If evaluation policy given by context is set to ``forbidden`` no evaluation will be performed; if it is set to ``dangerous`` standard :func:`eval` will be used; finally, for any other, policy :func:`eval_node` will be called on parsed AST. """ locals_ = context.locals if context.evaluation == "forbidden": raise GuardRejection("Forbidden mode") # note: not using `ast.literal_eval` as it does not implement # getitem at all, for example it fails on simple `[0][1]` if context.in_subscript: # syntatic sugar for ellipsis (:) is only available in susbcripts # so we need to trick the ast parser into thinking that we have # a subscript, but we need to be able to later recognise that we did # it so we can ignore the actual __getitem__ operation if not code: return tuple() locals_ = locals_.copy() locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT code = SUBSCRIPT_MARKER + "[" + code + "]" context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}}) if context.evaluation == "dangerous": return eval(code, context.globals, context.locals) expression = ast.parse(code, mode="eval") return eval_node(expression, context) def eval_node(node: Union[ast.AST, None], context: EvaluationContext): """Evaluate AST node in provided context. Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments. Does not evaluate actions that always have side effects: - class definitions (``class sth: ...``) - function definitions (``def sth: ...``) - variable assignments (``x = 1``) - augmented assignments (``x += 1``) - deletions (``del x``) Does not evaluate operations which do not return values: - assertions (``assert x``) - pass (``pass``) - imports (``import x``) - control flow: - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) - loops (``for`` and `while``) - exception handling The purpose of this function is to guard against unwanted side-effects; it does not give guarantees on protection from malicious code execution. """ policy = EVALUATION_POLICIES[context.evaluation] if node is None: return None if isinstance(node, ast.Expression): return eval_node(node.body, context) if isinstance(node, ast.BinOp): # TODO: add guards left = eval_node(node.left, context) right = eval_node(node.right, context) if isinstance(node.op, ast.Add): return left + right if isinstance(node.op, ast.Sub): return left - right if isinstance(node.op, ast.Mult): return left * right if isinstance(node.op, ast.Div): return left / right if isinstance(node.op, ast.FloorDiv): return left // right if isinstance(node.op, ast.Mod): return left % right if isinstance(node.op, ast.Pow): return left**right if isinstance(node.op, ast.LShift): return left << right if isinstance(node.op, ast.RShift): return left >> right if isinstance(node.op, ast.BitOr): return left | right if isinstance(node.op, ast.BitXor): return left ^ right if isinstance(node.op, ast.BitAnd): return left & right if isinstance(node.op, ast.MatMult): return left @ right if isinstance(node, ast.Constant): return node.value if isinstance(node, ast.Index): return eval_node(node.value, context) if isinstance(node, ast.Tuple): return tuple(eval_node(e, context) for e in node.elts) if isinstance(node, ast.List): return [eval_node(e, context) for e in node.elts] if isinstance(node, ast.Set): return {eval_node(e, context) for e in node.elts} if isinstance(node, ast.Dict): return dict( zip( [eval_node(k, context) for k in node.keys], [eval_node(v, context) for v in node.values], ) ) if isinstance(node, ast.Slice): return slice( eval_node(node.lower, context), eval_node(node.upper, context), eval_node(node.step, context), ) if isinstance(node, ast.ExtSlice): return tuple([eval_node(dim, context) for dim in node.dims]) if isinstance(node, ast.UnaryOp): # TODO: add guards value = eval_node(node.operand, context) if isinstance(node.op, ast.USub): return -value if isinstance(node.op, ast.UAdd): return +value if isinstance(node.op, ast.Invert): return ~value if isinstance(node.op, ast.Not): return not value raise ValueError("Unhandled unary operation:", node.op) if isinstance(node, ast.Subscript): value = eval_node(node.value, context) slice_ = eval_node(node.slice, context) if policy.can_get_item(value, slice_): return value[slice_] raise GuardRejection( "Subscript access (`__getitem__`) for", type(value), # not joined to avoid calling `repr` f" not allowed in {context.evaluation} mode", ) if isinstance(node, ast.Name): if policy.allow_locals_access and node.id in context.locals: return context.locals[node.id] if policy.allow_globals_access and node.id in context.globals: return context.globals[node.id] if policy.allow_builtins_access and hasattr(builtins, node.id): # note: do not use __builtins__, it is implementation detail of Python return getattr(builtins, node.id) if not policy.allow_globals_access and not policy.allow_locals_access: raise GuardRejection( f"Namespace access not allowed in {context.evaluation} mode" ) else: raise NameError(f"{node.id} not found in locals nor globals") if isinstance(node, ast.Attribute): value = eval_node(node.value, context) if policy.can_get_attr(value, node.attr): return getattr(value, node.attr) raise GuardRejection( "Attribute access (`__getattr__`) for", type(value), # not joined to avoid calling `repr` f"not allowed in {context.evaluation} mode", ) if isinstance(node, ast.IfExp): test = eval_node(node.test, context) if test: return eval_node(node.body, context) else: return eval_node(node.orelse, context) if isinstance(node, ast.Call): func = eval_node(node.func, context) if policy.can_call(func) and not node.keywords: args = [eval_node(arg, context) for arg in node.args] return func(*args) raise GuardRejection( "Call for", func, # not joined to avoid calling `repr` f"not allowed in {context.evaluation} mode", ) raise ValueError("Unhandled node", node) SUPPORTED_EXTERNAL_GETITEM = { ("pandas", "core", "indexing", "_iLocIndexer"), ("pandas", "core", "indexing", "_LocIndexer"), ("pandas", "DataFrame"), ("pandas", "Series"), ("numpy", "ndarray"), ("numpy", "void"), } BUILTIN_GETITEM: Set[InstancesHaveGetItem] = { dict, str, bytes, list, tuple, collections.defaultdict, collections.deque, collections.OrderedDict, collections.ChainMap, collections.UserDict, collections.UserList, collections.UserString, _DummyNamedTuple, _IdentitySubscript, } def _list_methods(cls, source=None): """For use on immutable objects or with methods returning a copy""" return [getattr(cls, k) for k in (source if source else dir(cls))] dict_non_mutating_methods = ("copy", "keys", "values", "items") list_non_mutating_methods = ("copy", "index", "count") set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) dict_keys: Type[collections.abc.KeysView] = type({}.keys()) method_descriptor: Any = type(list.copy) ALLOWED_CALLS = { bytes, *_list_methods(bytes), dict, *_list_methods(dict, dict_non_mutating_methods), dict_keys.isdisjoint, list, *_list_methods(list, list_non_mutating_methods), set, *_list_methods(set, set_non_mutating_methods), frozenset, *_list_methods(frozenset), range, str, *_list_methods(str), tuple, *_list_methods(tuple), collections.deque, *_list_methods(collections.deque, list_non_mutating_methods), collections.defaultdict, *_list_methods(collections.defaultdict, dict_non_mutating_methods), collections.OrderedDict, *_list_methods(collections.OrderedDict, dict_non_mutating_methods), collections.UserDict, *_list_methods(collections.UserDict, dict_non_mutating_methods), collections.UserList, *_list_methods(collections.UserList, list_non_mutating_methods), collections.UserString, *_list_methods(collections.UserString, dir(str)), collections.Counter, *_list_methods(collections.Counter, dict_non_mutating_methods), collections.Counter.elements, collections.Counter.most_common, } BUILTIN_GETATTR: Set[MayHaveGetattr] = { *BUILTIN_GETITEM, set, frozenset, object, type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. dict_keys, method_descriptor, } EVALUATION_POLICIES = { "minimal": EvaluationPolicy( allow_builtins_access=True, allow_locals_access=False, allow_globals_access=False, allow_item_access=False, allow_attr_access=False, allowed_calls=set(), allow_any_calls=False, ), "limited": SelectivePolicy( # TODO: # - should reject binary and unary operations if custom methods would be dispatched allowed_getitem=BUILTIN_GETITEM, allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, allowed_getattr=BUILTIN_GETATTR, allowed_getattr_external={ # pandas Series/Frame implements custom `__getattr__` ("pandas", "DataFrame"), ("pandas", "Series"), }, allow_builtins_access=True, allow_locals_access=True, allow_globals_access=True, allowed_calls=ALLOWED_CALLS, ), "unsafe": EvaluationPolicy( allow_builtins_access=True, allow_locals_access=True, allow_globals_access=True, allow_attr_access=True, allow_item_access=True, allow_any_calls=True, ), } __all__ = [ "guarded_eval", "eval_node", "GuardRejection", "EvaluationContext", "_unbind_method", ]