Blog

#!/usr/bin/env python3
"""은닉층 하나짜리 신경망의 순전파와 역전파를 직접 구현한다."""
import numpy as np

def sigmoid(z):
    return 1.0 / (1.0 + np.exp(-np.clip(z, -500, 500)))

def init(n_in, n_hidden, seed=0):
    rng = np.random.default_rng(seed)
    return {
        "W1": rng.normal(0, np.sqrt(1.0 / n_in), (n_in, n_hidden)),
        "b1": np.zeros(n_hidden),
        "W2": rng.normal(0, np.sqrt(1.0 / n_hidden), (n_hidden, 1)),
        "b2": np.zeros(1),
    }

def forward(P, X):
    z1 = X @ P["W1"] + P["b1"]      # (n, h)
    a1 = np.tanh(z1)
    z2 = a1 @ P["W2"] + P["b2"]     # (n, 1)
    p  = sigmoid(z2).ravel()
    return p, {"z1": z1, "a1": a1, "z2": z2, "p": p}

def loss(P, X, y):
    p = np.clip(forward(P, X)[0], 1e-12, 1 - 1e-12)
    return float(-np.mean(y * np.log(p) + (1 - y) * np.log(1 - p)))

def backward(P, X, y, c):
    """연쇄법칙을 층 순서 반대로 적용한다."""
    n = len(y)
    # 로그손실 + 시그모이드를 합치면 dL/dz2 = (p - y)/n 으로 깔끔하게 떨어진다
    dz2 = ((c["p"] - y) / n).reshape(-1, 1)          # (n, 1)
    dW2 = c["a1"].T @ dz2                             # (h, 1)
    db2 = dz2.sum(0)
    da1 = dz2 @ P["W2"].T                             # (n, h)
    dz1 = da1 * (1 - c["a1"] ** 2)                    # tanh' = 1 - tanh^2
    dW1 = X.T @ dz1
    db1 = dz1.sum(0)
    return {"W1": dW1, "b1": db1, "W2": dW2, "b2": db2}

#!/usr/bin/env python3
"""손으로 유도한 기울기를 수치미분과 대조한다 (gradient check)."""
import numpy as np
from mlp import init, forward, backward, loss

rng = np.random.default_rng(1)
X = rng.normal(size=(7, 5))
y = (rng.random(7) < 0.5).astype(float)
P = init(5, 4)

analytic = backward(P, X, y, forward(P, X)[1])

EPS = 1e-6
print(f"{'파라미터':<8}{'해석적 기울기':>16}{'수치미분':>16}{'상대오차':>12}")
print("-" * 54)
worst = 0.0
for name in ("W1", "b1", "W2", "b2"):
    A = analytic[name]
    it = np.nditer(P[name], flags=["multi_index"])
    for _ in it:
        i = it.multi_index
        orig = P[name][i]
        P[name][i] = orig + EPS; lp = loss(P, X, y)
        P[name][i] = orig - EPS; lm = loss(P, X, y)
        P[name][i] = orig
        num = (lp - lm) / (2 * EPS)
        rel = abs(num - A[i]) / max(1e-12, abs(num) + abs(A[i]))
        worst = max(worst, rel)
        if i in ((0, 0), (0,)):
            print(f"{name + str(i):<8}{A[i]:>16.9f}{num:>16.9f}{rel:>12.2e}")
print(f"\n전체 {sum(P[n].size for n in P)}개 파라미터의 최대 상대오차: {worst:.2e}")
print("판정:", "일치 (1e-7 미만)" if worst < 1e-7 else "불일치 — 역전파 식을 다시 볼 것")

#!/usr/bin/env python3
"""tanh 미분을 빠뜨린 역전파도 손실은 내려간다 — 수치미분만이 잡아낸다."""
import numpy as np
from data2 import load
from mlp import init, forward, backward, loss

def backward_bug(P, X, y, c):
    n = len(y)
    dz2 = ((c["p"] - y) / n).reshape(-1, 1)
    dW2 = c["a1"].T @ dz2
    db2 = dz2.sum(0)
    da1 = dz2 @ P["W2"].T
    dz1 = da1                                   # 버그: * (1 - a1**2) 누락
    return {"W1": X.T @ dz1, "b1": dz1.sum(0), "W2": dW2, "b2": db2}

NAMES = ["entropy", "bigram", "ascii", "uniq", "run"]
ext, F, ratio, tr, te = load()
y = (ratio < 0.30).astype(float)
X = np.column_stack([F[n] for n in NAMES])
Z = (X - X[tr].mean(0)) / X[tr].std(0)

print(f"{'스텝':>6}{'정상 학습손실':>14}{'버그 학습손실':>14}{'정상 평가':>10}{'버그 평가':>10}")
P_ok, P_bug = init(5, 2, 0), init(5, 2, 0)
for step in range(1, 4001):
    for P, bw in ((P_ok, backward), (P_bug, backward_bug)):
        g = bw(P, Z[tr], y[tr], forward(P, Z[tr])[1])
        for k in P:
            P[k] -= 0.5 * g[k]
    if step in (1, 10, 100, 1000, 4000):
        print(f"{step:>6}{loss(P_ok, Z[tr], y[tr]):>14.5f}{loss(P_bug, Z[tr], y[tr]):>14.5f}"
              f"{loss(P_ok, Z[te], y[te]):>10.5f}{loss(P_bug, Z[te], y[te]):>10.5f}")

# 같은 버그를 수치미분과 대조
rng = np.random.default_rng(1)
Xs, ys = rng.normal(size=(7, 5)), (rng.random(7) < 0.5).astype(float)
P = init(5, 4)
g = backward_bug(P, Xs, ys, forward(P, Xs)[1])
EPS, worst = 1e-6, {}
for name in P:
    it = np.nditer(P[name], flags=["multi_index"])
    for _ in it:
        i = it.multi_index; o = P[name][i]
        P[name][i] = o + EPS; lp = loss(P, Xs, ys)
        P[name][i] = o - EPS; lm = loss(P, Xs, ys)
        P[name][i] = o
        num = (lp - lm) / (2 * EPS)
        rel = abs(num - g[name][i]) / max(1e-12, abs(num) + abs(g[name][i]))
        worst[name] = max(worst.get(name, 0.0), rel)
print("\n버그 버전 gradient check — 파라미터별 최대 상대오차")
for name, v in worst.items():
    print(f"  {name:<3} {v:.2e}  {'불일치' if v > 1e-7 else '일치'}")

#!/usr/bin/env python3
"""직접 구현한 역전파로 실제 데이터를 학습시켜 로지스틱 회귀와 비교한다."""
import numpy as np
from data2 import load
from mlp import init, forward, backward, loss

NAMES = ["entropy", "bigram", "ascii", "uniq", "run"]
ext, F, ratio, tr, te = load()
y = (ratio < 0.30).astype(float)
X = np.column_stack([F[n] for n in NAMES])
mu, sd = X[tr].mean(0), X[tr].std(0)
Z = (X - mu) / sd

def train(hidden, lr=0.5, steps=4000, seed=0):
    P = init(Z.shape[1], hidden, seed)
    for _ in range(steps):
        p, c = forward(P, Z[tr])
        g = backward(P, Z[tr], y[tr], c)
        for k in P:
            P[k] -= lr * g[k]
    acc = ((forward(P, Z[te])[0] >= 0.5) == y[te]).mean()
    return loss(P, Z[tr], y[tr]), loss(P, Z[te], y[te]), acc

print(f"{'은닉 뉴런':>9}{'학습 로그손실':>14}{'평가 로그손실':>14}{'평가 정확도':>12}")
print("-" * 49)
for h in (1, 2, 4, 8, 16, 32):
    ltr, lte, acc = train(h)
    print(f"{h:>9}{ltr:>14.5f}{lte:>14.5f}{acc:>12.4f}")
print(f"{'(로지스틱)':>9}{0.36925:>14.5f}{0.36673:>14.5f}{0.8257:>12.4f}")

#!/usr/bin/env python3
"""층을 깊게 쌓았을 때 앞쪽 층으로 갈수록 기울기가 얼마나 줄어드는지 잰다."""
import numpy as np

def sigmoid(z):
    return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

ACT = {
    "sigmoid": (sigmoid, lambda a: a * (1 - a)),
    "tanh":    (np.tanh, lambda a: 1 - a ** 2),
    "relu":    (lambda z: np.maximum(z, 0), lambda a: (a > 0).astype(float)),
}

def layer_grad_norms(kind, depth=10, width=32, seed=0):
    fn, dfn = ACT[kind]
    rng = np.random.default_rng(seed)
    Ws = [rng.normal(0, np.sqrt(1.0 / width), (width, width)) for _ in range(depth)]
    x = rng.normal(size=(64, width))

    acts = [x]
    for W in Ws:
        acts.append(fn(acts[-1] @ W))
    delta = rng.normal(size=acts[-1].shape) / 64      # 출력단에서 내려온 기울기
    norms = []
    for i in reversed(range(depth)):
        delta = delta * dfn(acts[i + 1])
        norms.append(np.linalg.norm(acts[i].T @ delta))
        delta = delta @ Ws[i].T
    return norms[::-1]                                 # 입력쪽 층부터

print(f"{'층':>4}" + "".join(f"{k:>14}" for k in ACT))
print("-" * 46)
res = {k: layer_grad_norms(k) for k in ACT}
for i in range(10):
    print(f"{i + 1:>4}" + "".join(f"{res[k][i]:>14.3e}" for k in ACT))
print("\n1층 기울기 / 10층 기울기 비율")
for k in ACT:
    print(f"    {k:<8}: {res[k][0] / res[k][-1]:.3e}")