Summary  
This chapter demonstrates how to construct a dynamic computation graph in PyTorch with multiple intermediate tensors to perform multi-step backpropagation via the chain rule, and how to disable gradient tracking to save memory and computation.

General domain of usage  
Deep learning

Like **Tensorflow**, PyTorch also allows you to build more complex computational graphs involving **multiple intermediate tensors**.

import torch
# Create a 2D tensor with gradient tracking
x = torch.tensor([[1.0, 2.0, 3.0], [3.0, 2.0, 1.0]], requires_grad=True)
# Define intermediate layers
y = 6 * x + 3
z = 10 * y ** 2
# Compute the mean of the final output
output_mean = z.mean()
print(f"Output: {output_mean}")
# Perform backpropagation
output_mean.backward()
# Print the gradient of x
print("Gradient of x:\n", x.grad)

The gradient of `output_mean` with respect to `x` is computed using the **chain rule**. The result shows how much a small change in each element of `x` affects `output_mean`.

## Disabling Gradient Tracking

In some cases, you may want to disable gradient tracking to **save memory and computation**. Since `requires_grad=False` is the default behavior, you can simply create the tensor **without specifying this parameter**:
```python
x = torch.tensor([[1.0, 2.0, 3.0], [3.0, 2.0, 1.0]])
```

import unittest
import torch
import importlib
import io
import sys


def _dynamic(test, cond, success, fail):
    if cond:
        test._testMethodName = success
        test.assertTrue(True)
    else:
        test._testMethodName = fail
        test.fail(fail)


class TestUserCode(unittest.TestCase):

    # --------- Test W creation ---------
    def test_W_exists(self):
        import user_code
        cond = hasattr(user_code, "W")
        _dynamic(
            self,
            cond,
            "`W` is successfully defined.",
            "`W` is missing. Define the weight matrix using `torch.rand(1, 3, requires_grad=True)`."
        )

    def test_W_shape(self):
        import user_code
        W = user_code.W
        cond = isinstance(W, torch.Tensor) and W.shape == (1, 3)
        _dynamic(
            self,
            cond,
            "`W` has the correct shape (1x3).",
            "`W` must be a tensor of shape (1, 3)."
        )

    def test_W_requires_grad(self):
        import user_code
        W = user_code.W
        cond = hasattr(W, "requires_grad") and W.requires_grad is True
        _dynamic(
            self,
            cond,
            "`W` correctly tracks gradients (`requires_grad=True`).",
            "`W` must be created with `requires_grad=True` to enable gradient computation."
        )

    # --------- Test X ---------
    def test_X_correct(self):
        import user_code
        cond = hasattr(user_code, "X")
        if cond:
            expected = torch.tensor([[1.0, 2.0],
                                     [3.0, 4.0],
                                     [5.0, 6.0]])
            cond = torch.allclose(user_code.X.float(), expected)
        _dynamic(
            self,
            cond,
            "`X` is correctly defined from the given list.",
            "`X` must be created as `torch.tensor([[1.0,2.0],[3.0,4.0],[5.0,6.0]])`."
        )

    # --------- Test Y = W @ X ---------
    def test_Y_computed(self):
        import user_code
        cond = hasattr(user_code, "Y")
        _dynamic(
            self,
            cond,
            "`Y` is computed.",
            "`Y` is missing. Compute it using matrix multiplication: `W @ X`."
        )

    # --------- Test loss ---------
    def test_loss_computed(self):
        import user_code
        cond = hasattr(user_code, "loss")
        _dynamic(
            self,
            cond,
            "`loss` is successfully computed.",
            "`loss` is missing. Use mean squared error: `((Y - Y_target)**2).mean()`."
        )

    # --------- Test backward() ---------
    def test_backward_computed(self):
        import user_code
        try:
            W = user_code.W
            cond = W.grad is not None
        except:
            cond = False

        _dynamic(
            self,
            cond,
            "Gradient for `W` is computed using backpropagation.",
            "Gradient for `W` is missing. Make sure to call `loss.backward()`."
        )

    # --------- Test printed output ---------
    def test_output_printed(self):
        import user_code
        buf = io.StringIO()
        sys.stdout = buf

        importlib.reload(user_code)
        sys.stdout = sys.__stdout__

        cond = len(buf.getvalue().strip()) > 0
        _dynamic(
            self,
            cond,
            "Gradient values are printed.",
            "No gradient output detected. Make sure to print `W.grad`."
        )


if __name__ == "__main__":
    unittest.main()


test_code.py

Learn the fundamental and advanced concepts needed to work with PyTorch efficiently. Gain a solid understanding tensors, including creation, operations, and reshaping. Explore the essentials of gradients, backpropagation, and linear regression before moving on to handling datasets. Master the skill needed build, train, and evaluate neural networks.

Multi-Step Backpropagation

Disabling Gradient Tracking

Solution