{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"%matplotlib inline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\nPyTorch: optim\n--------------\n\nA fully-connected ReLU network with one hidden layer, trained to predict y from x\nby minimizing squared Euclidean distance.\n\nThis implementation uses the nn package from PyTorch to build the network.\n\nRather than manually updating the weights of the model as we have been doing,\nwe use the optim package to define an Optimizer that will update the weights\nfor us. The optim package defines many optimization algorithms that are commonly\nused for deep learning, including SGD+momentum, RMSProp, Adam, etc.\n\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import torch\n\n# N is batch size; D_in is input dimension;\n# H is hidden dimension; D_out is output dimension.\nN, D_in, H, D_out = 64, 1000, 100, 10\n\n# Create random Tensors to hold inputs and outputs\nx = torch.randn(N, D_in)\ny = torch.randn(N, D_out)\n\n# Use the nn package to define our model and loss function.\nmodel = torch.nn.Sequential(\n torch.nn.Linear(D_in, H),\n torch.nn.ReLU(),\n torch.nn.Linear(H, D_out),\n)\nloss_fn = torch.nn.MSELoss(reduction='sum')\n\n# Use the optim package to define an Optimizer that will update the weights of\n# the model for us. Here we will use Adam; the optim package contains many other\n# optimization algorithms. The first argument to the Adam constructor tells the\n# optimizer which Tensors it should update.\nlearning_rate = 1e-4\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\nfor t in range(500):\n # Forward pass: compute predicted y by passing x to the model.\n y_pred = model(x)\n\n # Compute and print loss.\n loss = loss_fn(y_pred, y)\n if t % 100 == 99:\n print(t, loss.item())\n\n # Before the backward pass, use the optimizer object to zero all of the\n # gradients for the variables it will update (which are the learnable\n # weights of the model). This is because by default, gradients are\n # accumulated in buffers( i.e, not overwritten) whenever .backward()\n # is called. Checkout docs of torch.autograd.backward for more details.\n optimizer.zero_grad()\n\n # Backward pass: compute gradient of the loss with respect to model\n # parameters\n loss.backward()\n\n # Calling the step function on an Optimizer makes an update to its\n # parameters\n optimizer.step()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 0
}