Source code for torchrl._utils

# Copyright (c) Meta Platforms, Inc. and affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import annotations

import collections

import functools
import inspect

import math
import os
import sys
import time
import traceback
import warnings
from copy import copy
from distutils.util import strtobool
from functools import wraps
from importlib import import_module
from typing import Any, Callable, cast, Dict, TypeVar, Union

import numpy as np
import torch
from packaging.version import parse
from torch import multiprocessing as mp

VERBOSE = strtobool(os.environ.get("VERBOSE", "0"))
_os_is_windows = sys.platform == "win32"
RL_WARNINGS = strtobool(os.environ.get("RL_WARNINGS", "1"))

class timeit:
    """A dirty but easy to use decorator for profiling code."""

    _REG = {}

    def __init__(self, name): = name

    def __call__(self, fn):
        def decorated_fn(*args, **kwargs):
            with self:
                out = fn(*args, **kwargs)
                return out

        return decorated_fn

    def __enter__(self):
        self.t0 = time.time()

    def __exit__(self, exc_type, exc_val, exc_tb):
        t = time.time() - self.t0
        val = self._REG.setdefault(, [0.0, 0.0, 0])

        count = val[2]
        N = count + 1
        val[0] = val[0] * (count / N) + t / N
        val[1] += t
        val[2] = N

    def print(prefix=None):
        keys = list(timeit._REG)
        for name in keys:
            strings = []
            if prefix:
                f"{name} took {timeit._REG[name][0] * 1000:4.4} msec (total = {timeit._REG[name][1]} sec)"
            print(" -- ".join(strings))

    def erase():
        for k in timeit._REG:
            timeit._REG[k] = [0.0, 0.0, 0]

def _check_for_faulty_process(processes):
    terminate = False
    for p in processes:
        if not p.is_alive():
            terminate = True
            for _p in processes:
                if _p.is_alive():
        if terminate:
    if terminate:
        raise RuntimeError(
            "At least one process failed. Check for more infos in the log."

def seed_generator(seed):
    """A seed generator function.

    Given a seeding integer, generates a deterministic next seed to be used in a
    seeding sequence.

        seed (int): initial seed.

    Returns: Next seed of the chain.

    max_seed_val = (
        2**32 - 1
    )  #
    rng = np.random.default_rng(seed)
    seed = int.from_bytes(rng.bytes(8), "big")
    return seed % max_seed_val

class KeyDependentDefaultDict(collections.defaultdict):
    """A key-dependent default dict.

        >>> my_dict = KeyDependentDefaultDict(lambda key: "foo_" + key)
        >>> print(my_dict["bar"])

    def __init__(self, fun): = fun

    def __missing__(self, key):
        value =
        self[key] = value
        return value

def prod(sequence):
    """General prod function, that generalised usage across math and np.

    Created for multiple python versions compatibility).

    if hasattr(math, "prod"):
        return int(

def get_binary_env_var(key):
    """Parses and returns the binary environment variable value.

    If not present in environment, it is considered `False`.

        key (str): name of the environment variable.
    val = os.environ.get(key, "False")
    if val in ("0", "False", "false"):
        val = False
    elif val in ("1", "True", "true"):
        val = True
        raise ValueError(
            f"Environment variable {key} should be in 'True', 'False', '0' or '1'. "
            f"Got {val} instead."
    return val

class _Dynamic_CKPT_BACKEND:
    """Allows CKPT_BACKEND to be changed on-the-fly."""

    backends = ["torch", "torchsnapshot"]

    def _get_backend(self):
        backend = os.environ.get("CKPT_BACKEND", "torchsnapshot")
        if backend == "torchsnapshot":
                import torchsnapshot  # noqa: F401

                _has_ts = True
            except ImportError:
                _has_ts = False
            if not _has_ts:
                raise ImportError(
                    f"torchsnapshot not found, but the backend points to this library. "
                    f"Consider installing torchsnapshot or choose another backend (available backends: {self.backends})"
        return backend

    def __getattr__(self, item):
        return getattr(self._get_backend(), item)

    def __eq__(self, other):
        return self._get_backend() == other

    def __ne__(self, other):
        return self._get_backend() != other

    def __repr__(self):
        return self._get_backend()


[docs]class implement_for: """A version decorator that checks the version in the environment and implements a function with the fitting one. If specified module is missing or there is no fitting implementation, call of the decorated function will lead to the explicit error. In case of intersected ranges, last fitting implementation is used. This wrapper also works to implement different backends for a same function (eg. gym vs gymnasium, numpy vs jax-numpy etc). Args: module_name (str or callable): version is checked for the module with this name (e.g. "gym"). If a callable is provided, it should return the module. from_version: version from which implementation is compatible. Can be open (None). to_version: version from which implementation is no longer compatible. Can be open (None). Examples: >>> @implement_for("gym", "0.13", "0.14") >>> def fun(self, x): ... # Older gym versions will return x + 1 ... return x + 1 ... >>> @implement_for("gym", "0.14", "0.23") >>> def fun(self, x): ... # More recent gym versions will return x + 2 ... return x + 2 ... >>> @implement_for(lambda: import_module("gym"), "0.23", None) >>> def fun(self, x): ... # More recent gym versions will return x + 2 ... return x + 2 ... >>> @implement_for("gymnasium", "0.27", None) >>> def fun(self, x): ... # If gymnasium is to be used instead of gym, x+3 will be returned ... return x + 3 ... This indicates that the function is compatible with gym 0.13+, but doesn't with gym 0.14+. """ # Stores pointers to fitting implementations: dict[func_name] = func_pointer _implementations = {} _setters = [] _cache_modules = {} def __init__( self, module_name: Union[str, Callable], from_version: str = None, to_version: str = None, ): self.module_name = module_name self.from_version = from_version self.to_version = to_version implement_for._setters.append(self) @staticmethod def check_version(version, from_version, to_version): return (from_version is None or parse(version) >= parse(from_version)) and ( to_version is None or parse(version) < parse(to_version) )
[docs] @staticmethod def get_class_that_defined_method(f): """Returns the class of a method, if it is defined, and None otherwise.""" out = f.__globals__.get(f.__qualname__.split(".")[0], None) return out
@classmethod def get_func_name(cls, fn): # produces a name like torchrl.module.Class.method or torchrl.module.function first = str(fn).split(".")[0][len("<function ") :] last = str(fn).split(".")[1:] if last: first = [first] last[-1] = last[-1].split(" ")[0] else: last = [first.split(" ")[0]] first = [] return ".".join([fn.__module__] + first + last) def _get_cls(self, fn): cls = self.get_class_that_defined_method(fn) if cls is None: # class not yet defined return if cls.__class__.__name__ == "function": cls = inspect.getmodule(fn) return cls
[docs] def module_set(self): """Sets the function in its module, if it exists already.""" prev_setter = type(self)._implementations.get(self.get_func_name(self.fn), None) if prev_setter is not None: prev_setter.do_set = False type(self)._implementations[self.get_func_name(self.fn)] = self cls = self.get_class_that_defined_method(self.fn) if cls is not None: if cls.__class__.__name__ == "function": cls = inspect.getmodule(self.fn) else: # class not yet defined return setattr(cls, self.fn.__name__, self.fn)
[docs] @classmethod def import_module(cls, module_name: Union[Callable, str]) -> str: """Imports module and returns its version.""" if not callable(module_name): module = cls._cache_modules.get(module_name, None) if module is None: if module_name in sys.modules: sys.modules[module_name] = module = import_module(module_name) else: cls._cache_modules[module_name] = module = import_module( module_name ) else: module = module_name() return module.__version__
_lazy_impl = collections.defaultdict(list) def _delazify(self, func_name): for local_call in implement_for._lazy_impl[func_name]: out = local_call() return out def __call__(self, fn): # function names are unique self.func_name = self.get_func_name(fn) self.fn = fn implement_for._lazy_impl[self.func_name].append(self._call) @wraps(fn) def _lazy_call_fn(*args, **kwargs): # first time we call the function, we also do the replacement. # This will cause the imports to occur only during the first call to fn return self._delazify(self.func_name)(*args, **kwargs) return _lazy_call_fn def _call(self): # If the module is missing replace the function with the mock. fn = self.fn func_name = self.func_name implementations = implement_for._implementations @wraps(fn) def unsupported(*args, **kwargs): raise ModuleNotFoundError( f"Supported version of '{func_name}' has not been found." ) self.do_set = False # Return fitting implementation if it was encountered before. if func_name in implementations: try: # check that backends don't conflict version = self.import_module(self.module_name) if self.check_version(version, self.from_version, self.to_version): if VERBOSE: module = import_module(self.module_name) warnings.warn( f"Got multiple backends for {func_name}. " f"Using the last queried ({module} with version {version})." ) self.do_set = True if not self.do_set: return implementations[func_name].fn except ModuleNotFoundError: # then it's ok, there is no conflict return implementations[func_name].fn else: try: version = self.import_module(self.module_name) if self.check_version(version, self.from_version, self.to_version): self.do_set = True except ModuleNotFoundError: return unsupported if self.do_set: self.module_set() return fn return unsupported
[docs] @classmethod def reset(cls, setters_dict: Dict[str, implement_for] = None): """Resets the setters in setter_dict. ``setter_dict`` is a copy of implementations. We just need to iterate through its values and call :meth:`~.module_set` for each. """ if VERBOSE: print("resetting implement_for") if setters_dict is None: setters_dict = copy(cls._implementations) for setter in setters_dict.values(): setter.module_set()
def __repr__(self): return ( f"{self.__class__.__name__}(" f"module_name={self.module_name}({self.from_version, self.to_version}), " f"fn_name={self.fn.__name__}, cls={self._get_cls(self.fn)}, is_set={self.do_set})" )
def accept_remote_rref_invocation(func): """Decorator that allows a method to be invoked remotely. Passes the `rpc.RRef` associated with the remote object construction as first argument in place of the object reference. """ @wraps(func) def unpack_rref_and_invoke_function(self, *args, **kwargs): # windows does not know torch._C._distributed_rpc.PyRRef if not _os_is_windows and isinstance(self, torch._C._distributed_rpc.PyRRef): self = self.local_value() return func(self, *args, **kwargs) return unpack_rref_and_invoke_function def accept_remote_rref_udf_invocation(decorated_class): """Class decorator that applies `accept_remote_rref_invocation` to all public methods.""" # ignores private methods for name in dir(decorated_class): method = getattr(decorated_class, name) if callable(method) and not name.startswith("_"): setattr(decorated_class, name, accept_remote_rref_invocation(method)) return decorated_class # We copy this from torch as older versions do not have it # see torch.utils._contextlib # Extra utilities for working with context managers that should have been # in the standard library but are not # Used for annotating the decorator usage of _DecoratorContextManager (e.g., # 'no_grad' and 'enable_grad'). # See FuncType = Callable[..., Any] F = TypeVar("F", bound=FuncType) def _wrap_generator(ctx_factory, func): """Wrap each generator invocation with the context manager factory. The input should be a function that returns a context manager, not a context manager itself, to handle one-shot context managers. """ @functools.wraps(func) def generator_context(*args, **kwargs): gen = func(*args, **kwargs) # Generators are suspended and unsuspended at `yield`, hence we # make sure the grad mode is properly set every time the execution # flow returns into the wrapped generator and restored when it # returns through our `yield` to our caller (see PR #49017). try: # Issuing `None` to a generator fires it up with ctx_factory(): response = gen.send(None) while True: try: # Forward the response to our caller and get its next request request = yield response except GeneratorExit: # Inform the still active generator about its imminent closure with ctx_factory(): gen.close() raise except BaseException: # Propagate the exception thrown at us by the caller with ctx_factory(): response = gen.throw(*sys.exc_info()) else: # Pass the last request to the generator and get its response with ctx_factory(): response = gen.send(request) # We let the exceptions raised above by the generator's `.throw` or # `.send` methods bubble up to our caller, except for StopIteration except StopIteration as e: # The generator informed us that it is done: take whatever its # returned value (if any) was and indicate that we're done too # by returning it (see docs for python's return-statement). return e.value return generator_context def context_decorator(ctx, func): """Context decorator. Like contextlib.ContextDecorator, but: 1. Is done by wrapping, rather than inheritance, so it works with context managers that are implemented from C and thus cannot easily inherit from Python classes 2. Wraps generators in the intuitive way (c.f. 3. Errors out if you try to wrap a class, because it is ambiguous whether or not you intended to wrap only the constructor The input argument can either be a context manager (in which case it must be a multi-shot context manager that can be directly invoked multiple times) or a callable that produces a context manager. """ if callable(ctx) and hasattr(ctx, "__enter__"): raise RuntimeError( f"Passed in {ctx} is both callable and also a valid context manager " "(has __enter__), making it ambiguous which interface to use. If you " "intended to pass a context manager factory, rewrite your call as " "context_decorator(lambda: ctx()); if you intended to pass a context " "manager directly, rewrite your call as context_decorator(lambda: ctx)" ) if not callable(ctx): def ctx_factory(): return ctx else: ctx_factory = ctx if inspect.isclass(func): raise RuntimeError( "Cannot decorate classes; it is ambiguous whether or not only the " "constructor or all methods should have the context manager applied; " "additionally, decorating a class at definition-site will prevent " "use of the identifier as a conventional type. " "To specify which methods to decorate, decorate each of them " "individually." ) if inspect.isgeneratorfunction(func): return _wrap_generator(ctx_factory, func) @functools.wraps(func) def decorate_context(*args, **kwargs): with ctx_factory(): return func(*args, **kwargs) return decorate_context class _DecoratorContextManager: """Allow a context manager to be used as a decorator.""" def __call__(self, orig_func: F) -> F: if inspect.isclass(orig_func): warnings.warn( "Decorating classes is deprecated and will be disabled in " "future versions. You should only decorate functions or methods. " "To preserve the current behavior of class decoration, you can " "directly decorate the `__init__` method and nothing else." ) func = cast(F, lambda *args, **kwargs: orig_func(*args, **kwargs)) else: func = orig_func return cast(F, context_decorator(self.clone, func)) def __enter__(self) -> None: raise NotImplementedError def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: raise NotImplementedError def clone(self): # override this method if your children class takes __init__ parameters return self.__class__() def get_trace(): """A simple debugging util to spot where a function is being called.""" traceback.print_stack() class _ProcessNoWarn(mp.Process): """A private Process class that shuts down warnings on the subprocess and controls the number of threads in the subprocess.""" @wraps(mp.Process.__init__) def __init__(self, *args, num_threads=None, **kwargs): import torchrl self.filter_warnings_subprocess = torchrl.filter_warnings_subprocess self.num_threads = num_threads super().__init__(*args, **kwargs) def run(self, *args, **kwargs): if self.num_threads is not None: torch.set_num_threads(self.num_threads) if self.filter_warnings_subprocess: import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") return, *args, **kwargs) return, *args, **kwargs)


Access comprehensive developer documentation for PyTorch

View Docs


Get in-depth tutorials for beginners and advanced developers

View Tutorials


Find development resources and get your questions answered

View Resources