MIT 6.7960 PS1 實作走查：從 NumPy 手寫 MLP 到 PyTorch Autograd 反向傳播

🌏 English version

MIT 6.7960 Fall 2024 OCW Homework 1 是課程的第一個編程作業，核心目標：從零實作 MLP 的前向與反向傳播，理解自動微分的底層機制。這篇文章完整走查 HW1 的每個考點，提供可直接執行的 NumPy 參考實作與 PyTorch 驗證腳本。

HW1 題目架構概覽

HW1 包含四個主要部分（對應 PDF 題目）：

1. Problem 1: Softmax & Cross-Entropy — 推導向量化公式、數值穩定 log-sum-exp
2. Problem 2: Two-Layer MLP Forward — 矩陣乘法、ReLU、Softmax 串接
3. Problem 3: Backpropagation by Hand — 鏈式法則逐層推導 ∂L/∂W、∂L/∂b
4. Problem 4: Gradient Check — Finite-difference 數值驗證解析梯度

評分重點：不只跑得出來，還要能解釋每一步梯度從哪來、為什麼這麼寫。

核心數學推導：兩層 MLP 反向傳播

網路架構：x → W1,b1 → ReLU → W2,b2 → Softmax → CE Loss

Forward:
z1 = x @ W1.T + b1          # [B, H]
a1 = ReLU(z1)               # [B, H]
z2 = a1 @ W2.T + b2         # [B, C]
probs = Softmax(z2)         # [B, C]
L = CE(probs, y)            # scalar

Backward (鏈式法則):
∂L/∂z2 = probs - y_onehot   # [B, C]  ← Softmax+CE 合併導數的經典結果
∂L/∂W2 = (∂L/∂z2).T @ a1    # [C, H]
∂L/∂b2 = sum(∂L/∂z2, dim=0) # [C]
∂L/∂a1 = ∂L/∂z2 @ W2        # [B, H]
∂L/∂z1 = ∂L/∂a1 * (z1 > 0)  # [B, H]  ← ReLU 導數
∂L/∂W1 = (∂L/∂z1).T @ x     # [H, D]
∂L/∂b1 = sum(∂L/∂z1, dim=0) # [H]

關鍵洞見：Softmax + CrossEntropy 的導數合併為 probs - y_onehot，這是因為：

∂CE/∂z2_i = p_i - 1{y=i}

這避免了數值不穩的 log(softmax) 顯式計算。

NumPy 參考實作：從零手寫 MLP

"""HW1 參考實作：NumPy 兩層 MLP + 手寫反向傳播 + Gradient Check"""
import numpy as np

def softmax(z):
    """數值穩定 Softmax"""
    z_max = np.max(z, axis=1, keepdims=True)
    exp_z = np.exp(z - z_max)
    return exp_z / np.sum(exp_z, axis=1, keepdims=True)

def cross_entropy_loss(probs, y):
    """向量化 CE Loss，y: [B] 類別索引"""
    B = probs.shape[0]
    log_probs = -np.log(probs[np.arange(B), y] + 1e-12)
    return np.mean(log_probs)

def relu(x):
    return np.maximum(0, x)

def relu_grad(z):
    return (z > 0).astype(float)

class TwoLayerMLP:
    def __init__(self, input_dim, hidden_dim, output_dim, seed=42):
        np.random.seed(seed)
        # He 初始化
        self.W1 = np.random.randn(hidden_dim, input_dim) * np.sqrt(2.0 / input_dim)
        self.b1 = np.zeros(hidden_dim)
        self.W2 = np.random.randn(output_dim, hidden_dim) * np.sqrt(2.0 / hidden_dim)
        self.b2 = np.zeros(output_dim)
    
    def forward(self, x):
        """x: [B, D]"""
        self.x = x
        self.z1 = x @ self.W1.T + self.b1      # [B, H]
        self.a1 = relu(self.z1)                 # [B, H]
        self.z2 = self.a1 @ self.W2.T + self.b2 # [B, C]
        self.probs = softmax(self.z2)           # [B, C]
        return self.probs
    
    def loss(self, y):
        return cross_entropy_loss(self.probs, y)
    
    def backward(self, y):
        B = self.x.shape[0]
        y_onehot = np.zeros_like(self.probs)
        y_onehot[np.arange(B), y] = 1.0
        
        # 輸出層梯度
        dz2 = self.probs - y_onehot             # [B, C]
        self.dW2 = dz2.T @ self.a1 / B          # [C, H]
        self.db2 = np.mean(dz2, axis=0)         # [C]
        
        # 隱藏層梯度
        da1 = dz2 @ self.W2                     # [B, H]
        dz1 = da1 * relu_grad(self.z1)          # [B, H]
        self.dW1 = dz1.T @ self.x / B           # [H, D]
        self.db1 = np.mean(dz1, axis=0)         # [H]
        
        return dict(dW1=self.dW1, db1=self.db1, dW2=self.dW2, db2=self.db2)
    
    def params_and_grads(self):
        return [
            (self.W1, self.dW1), (self.b1, self.db1),
            (self.W2, self.dW2), (self.b2, self.db2)
        ]

def finite_difference_check(model, x, y, eps=1e-5, tol=1e-4):
    """數值梯度檢查：比對解析梯度 vs 有限差分"""
    model.forward(x)
    model.backward(y)
    analytical = {name: grad.copy() for name, grad in 
                  [('W1', model.dW1), ('b1', model.db1), 
                   ('W2', model.dW2), ('b2', model.db2)]}
    
    max_rel_error = 0.0
    for name, param in [('W1', model.W1), ('b1', model.b1), 
                         ('W2', model.W2), ('b2', model.b2)]:
        grad_num = np.zeros_like(param)
        it = np.nditer(param, flags=['multi_index'], op_flags=['readwrite'])
        while not it.finished:
            idx = it.multi_index
            old_val = param[idx]
            
            param[idx] = old_val + eps
            loss_plus = model.loss(model.forward(x))
            
            param[idx] = old_val - eps
            loss_minus = model.loss(model.forward(x))
            
            param[idx] = old_val
            grad_num[idx] = (loss_plus - loss_minus) / (2 * eps)
            it.iternext()
        
        grad_ana = analytical[name]
        rel_error = np.max(np.abs(grad_num - grad_ana) / (np.abs(grad_num) + np.abs(grad_ana) + 1e-10))
        max_rel_error = max(max_rel_error, rel_error)
        print(f"{name}: max rel error = {rel_error:.2e} {'✓' if rel_error < tol else '✗ FAIL'}")
    
    print(f"Overall max rel error: {max_rel_error:.2e}")
    return max_rel_error < tol

# 執行驗證
if __name__ == "__main__":
    # 合成資料
    np.random.seed(123)
    B, D, H, C = 64, 20, 32, 5
    x = np.random.randn(B, D)
    y = np.random.randint(0, C, B)
    
    model = TwoLayerMLP(D, H, C)
    probs = model.forward(x)
    loss = model.loss(y)
    print(f"Initial loss: {loss:.4f}")
    
    # 梯度檢查
    print("\n=== Gradient Check ===")
    finite_difference_check(model, x, y)
    
    # 訓練幾步驗證收斂
    print("\n=== Training Steps ===")
    lr = 0.1
    for step in range(10):
        model.forward(x)
        loss = model.loss(y)
        model.backward(y)
        for param, grad in model.params_and_grads():
            param -= lr * grad
        if step % 2 == 0:
            print(f"Step {step}: loss = {loss:.4f}")

PyTorch Autograd 驗證：確保數值一致

"""PyTorch 版本：同樣架構，用 Autograd 自動反傳，比對數值"""
import torch
import torch.nn as nn
import torch.nn.functional as F

class TorchMLP(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super().__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim, bias=True)
        self.fc2 = nn.Linear(hidden_dim, output_dim, bias=True)
        # 複製 NumPy 的 He 初始化
        nn.init.kaiming_normal_(self.fc1.weight, nonlinearity='relu')
        nn.init.zeros_(self.fc1.bias)
        nn.init.kaiming_normal_(self.fc2.weight, nonlinearity='relu')
        nn.init.zeros_(self.fc2.bias)
    
    def forward(self, x):
        x = F.relu(self.fc1(x))
        return self.fc2(x)  # logits，不做 softmax（CrossEntropyLoss 內含）

def compare_numpy_torch():
    torch.manual_seed(123)
    np.random.seed(123)
    
    B, D, H, C = 64, 20, 32, 5
    x_np = np.random.randn(B, D).astype(np.float32)
    y_np = np.random.randint(0, C, B)
    
    x_torch = torch.from_numpy(x_np).requires_grad_(False)
    y_torch = torch.from_numpy(y_np).long()
    
    # NumPy 模型
    np_model = TwoLayerMLP(D, H, C, seed=123)
    
    # PyTorch 模型（複製權重）
    torch_model = TorchMLP(D, H, C)
    torch_model.fc1.weight.data = torch.from_numpy(np_model.W1.copy())
    torch_model.fc1.bias.data = torch.from_numpy(np_model.b1.copy())
    torch_model.fc2.weight.data = torch.from_numpy(np_model.W2.copy())
    torch_model.fc2.bias.data = torch.from_numpy(np_model.b2.copy())
    
    # 前向比對
    np_probs = np_model.forward(x_np)
    torch_logits = torch_model(x_torch)
    torch_probs = F.softmax(torch_logits, dim=1).detach().numpy()
    
    print(f"Forward max diff: {np.max(np.abs(np_probs - torch_probs)):.2e}")
    
    # 損失比對
    np_loss = np_model.loss(y_np)
    torch_loss = F.cross_entropy(torch_logits, y_torch).item()
    print(f"Loss diff: {abs(np_loss - torch_loss):.2e}")
    
    # 反向比對
    np_model.backward(y_np)
    torch_loss.backward()
    
    for name, (np_param, torch_param) in [
        ('W1', (np_model.W1, torch_model.fc1.weight)),
        ('b1', (np_model.b1, torch_model.fc1.bias)),
        ('W2', (np_model.W2, torch_model.fc2.weight)),
        ('b2', (np_model.b2, torch_model.fc2.bias))
    ]:
        np_grad = getattr(np_model, f'd{name}')
        torch_grad = torch_param.grad.numpy()
        rel_err = np.max(np.abs(np_grad - torch_grad) / (np.abs(np_grad) + np.abs(torch_grad) + 1e-10))
        print(f"{name} grad rel error: {rel_err:.2e} {'✓' if rel_err < 1e-5 else '✗'}")

if __name__ == "__main__":
    compare_numpy_torch()

常見錯誤與除錯清單

影片時間戳（相關講義）

• Lec 01: Introduction (YouTube: 6FkRvTtUc-o) — MLP 架構概覽
• Lec 02: How to Train a Neural Net (YouTube: vidCX_dMCu0) — 反向傳播推導、鏈式法則
• PyTorch Tutorial (YouTube: o5gPABcGZwc) — Autograd 基礎操作

擴充練習：自己加上這些功能

1. Mini-batch SGD：把全批次改成 DataLoader 迭代
2. Momentum / Adam：自己手寫優化器類別
3. Learning rate schedule：Cosine decay 實作
4. Weight decay：在損失函數加 L2 或解耦實作
5. Dropout / BatchNorm：前向/反向都要正確處理

參考資料

• MIT 6.7960 Fall 2024 Homework 1 (PDF) — 官方題目
• Lec 01: Introduction to Deep Learning — MLP 基礎
• Lec 02: How to Train a Neural Net — 反向傳播詳解
• PyTorch Tutorial (OCW) — Autograd 入門
• CS231n: Backpropagation Notes — 經典反向傳播教學（Karpathy）
• The Matrix Calculus You Need For Deep Learning (Parr & Howard, 2018) — 矩陣微分速查
• PyTorch Autograd 官方文件 — 機制與 hooks
• 站內：MIT 6.7960 L03：優化總覽 — 優化器設定
• 站內：MIT 6.7960 L04：正則化實戰 — 正則化加強版