[docs]@_use_grad_for_differentiabledefstep(self,closure=None):"""Performs a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. """loss=NoneifclosureisnotNone:withtorch.enable_grad():loss=closure()forgroupinself.param_groups:params_with_grad=[]grads=[]state_sums=[]state_steps=[]has_sparse_grad=self._init_group(group,params_with_grad,grads,state_sums,state_steps)adagrad(params_with_grad,grads,state_sums,state_steps,lr=group["lr"],weight_decay=group["weight_decay"],lr_decay=group["lr_decay"],eps=group["eps"],has_sparse_grad=has_sparse_grad,foreach=group["foreach"],maximize=group["maximize"],differentiable=group["differentiable"],)returnloss
Adagrad.__doc__=r"""Implements Adagrad algorithm. .. math:: \begin{aligned} &\rule{110mm}{0.4pt} \\ &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ &\hspace{12mm} \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\ &\textbf{initialize} : state\_sum_0 \leftarrow 0 \\[-1.ex] &\rule{110mm}{0.4pt} \\ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm} \tilde{\gamma} \leftarrow \gamma / (1 +(t-1) \eta) \\ &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ &\hspace{5mm}state\_sum_t \leftarrow state\_sum_{t-1} + g^2_t \\ &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon} \\ &\rule{110mm}{0.4pt} \\[-1.ex] &\bf{return} \: \theta_t \\[-1.ex] &\rule{110mm}{0.4pt} \\[-1.ex] \end{aligned} For further details regarding the algorithm we refer to `Adaptive Subgradient Methods for Online Learning and Stochastic Optimization`_. """+fr""" Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lr_decay (float, optional): learning rate decay (default: 0) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-10){_foreach_doc}{_maximize_doc}{_differentiable_doc} .. _Adaptive Subgradient Methods for Online Learning and Stochastic Optimization: http://jmlr.org/papers/v12/duchi11a.html """defadagrad(params:List[Tensor],grads:List[Tensor],state_sums:List[Tensor],state_steps:List[Tensor],# kwonly args with defaults are not supported by functions compiled with torchscript issue #70627# setting these as kwargs for now as functional API is compiled by torch/distributed/optimhas_sparse_grad:bool=None,foreach:Optional[bool]=None,differentiable:bool=False,*,lr:float,weight_decay:float,lr_decay:float,eps:float,maximize:bool,):r"""Functional API that performs Adagrad algorithm computation. See :class:`~torch.optim.Adagrad` for details. """ifnotall(isinstance(t,torch.Tensor)fortinstate_steps):raiseRuntimeError("API has changed, `state_steps` argument must contain a list of singleton tensors")ifforeachisNone:_,foreach=_default_to_fused_or_foreach(params,differentiable,use_fused=False)ifforeachandtorch.jit.is_scripting():raiseRuntimeError("torch.jit.script not supported with foreach optimizers")ifforeachandnottorch.jit.is_scripting():func=_multi_tensor_adagradelse:func=_single_tensor_adagradfunc(params,grads,state_sums,state_steps,lr=lr,weight_decay=weight_decay,lr_decay=lr_decay,eps=eps,has_sparse_grad=has_sparse_grad,maximize=maximize,differentiable=differentiable,)def_make_sparse(grad,grad_indices,values):size=grad.size()ifgrad_indices.numel()==0orvalues.numel()==0:returntorch.empty_like(grad)returntorch.sparse_coo_tensor(grad_indices,values,size)def_single_tensor_adagrad(params:List[Tensor],grads:List[Tensor],state_sums:List[Tensor],state_steps:List[Tensor],*,lr:float,weight_decay:float,lr_decay:float,eps:float,has_sparse_grad:bool,maximize:bool,differentiable:bool,):for(param,grad,state_sum,step_t)inzip(params,grads,state_sums,state_steps):# update stepstep_t+=1step=_get_value(step_t)grad=gradifnotmaximizeelse-gradifweight_decay!=0:ifgrad.is_sparse:raiseRuntimeError("weight_decay option is not compatible with sparse gradients")grad=grad.add(param,alpha=weight_decay)clr=lr/(1+(step-1)*lr_decay)ifgrad.is_sparse:grad=grad.coalesce()# the update is non-linear so indices must be uniquegrad_indices=grad._indices()grad_values=grad._values()state_sum.add_(_make_sparse(grad,grad_indices,grad_values.pow(2)))std=state_sum.sparse_mask(grad)std_values=std._values().sqrt_().add_(eps)param.add_(_make_sparse(grad,grad_indices,grad_values/std_values),alpha=-clr)else:is_complex=torch.is_complex(param)ifis_complex:grad=torch.view_as_real(grad)state_sum=torch.view_as_real(state_sum)param=torch.view_as_real(param)state_sum.addcmul_(grad,grad,value=1)ifdifferentiable:std=state_sum.sqrt()+epselse:std=state_sum.sqrt().add_(eps)param.addcdiv_(grad,std,value=-clr)ifis_complex:param=torch.view_as_complex(param)state_sum=torch.view_as_complex(state_sum)def_multi_tensor_adagrad(params:List[Tensor],grads:List[Tensor],state_sums:List[Tensor],state_steps:List[Tensor],*,lr:float,weight_decay:float,lr_decay:float,eps:float,has_sparse_grad:bool,maximize:bool,differentiable:bool,):assertnotdifferentiable,"_foreach ops don't support autograd"# Foreach functions will throw errors if given empty listsiflen(params)==0:returngrouped_tensorlists=Optimizer._group_tensors_by_device_and_dtype([params,grads,state_sums,state_steps])for((device_params,device_grads,device_state_sums,device_state_steps),_)ingrouped_tensorlists.values():device_has_sparse_grad=any(grad.is_sparseforgradindevice_grads)ifdevice_has_sparse_grad:return_single_tensor_adagrad(device_params,device_grads,device_state_sums,device_state_steps,lr=lr,weight_decay=weight_decay,lr_decay=lr_decay,eps=eps,has_sparse_grad=True,maximize=False,differentiable=differentiable,)ifmaximize:device_grads=torch._foreach_neg(device_grads)# Handle complex parametersdevice_grads=[torch.view_as_real(x)iftorch.is_complex(x)elsexforxindevice_grads]device_state_sums=[torch.view_as_real(x)iftorch.is_complex(x)elsexforxindevice_state_sums]device_params=[torch.view_as_real(x)iftorch.is_complex(x)elsexforxindevice_params]# Update stepstorch._foreach_add_(device_state_steps,1)ifweight_decay!=0:# Re-use the intermediate memory (device_grads) already allocated for maximizeifmaximize:torch._foreach_add_(device_grads,device_params,alpha=weight_decay)else:device_grads=torch._foreach_add(device_grads,device_params,alpha=weight_decay)minus_clr=[-lr/(1+(_get_value(step)-1)*lr_decay)forstepindevice_state_steps]torch._foreach_addcmul_(device_state_sums,device_grads,device_grads,value=1)std=torch._foreach_sqrt(device_state_sums)torch._foreach_add_(std,eps)ifweight_decay!=0ormaximize:# Again, re-use the intermediate memory (device_grads) already allocatedtorch._foreach_mul_(device_grads,minus_clr)numerator=device_gradselse:numerator=torch._foreach_mul(device_grads,minus_clr)torch._foreach_addcdiv_(device_params,numerator,std)
Docs
Access comprehensive developer documentation for PyTorch
To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.