Blog

"""files.csv를 읽어 학습/평가 분할까지 만들어 두는 공용 모듈."""
import csv
import numpy as np

def load(path="files.csv", seed=0, train_ratio=0.7):
    rows    = list(csv.DictReader(open(path)))
    ext     = np.array([r["ext"] for r in rows])
    size    = np.array([int(r["size"]) for r in rows], dtype=float)
    entropy = np.array([float(r["entropy"]) for r in rows])
    ratio   = np.array([int(r["gz_size"]) for r in rows]) / size

    idx = np.random.default_rng(seed).permutation(len(rows))
    n_train = int(len(rows) * train_ratio)
    return ext, size, entropy, ratio, idx[:n_train], idx[n_train:]

#!/usr/bin/env python3
"""최소제곱을 정규방정식으로 직접 풀고 np.linalg.lstsq와 비교한다."""
import numpy as np
from data import load

ext, size, entropy, ratio, tr, te = load()

X = np.column_stack([entropy[tr], np.ones(len(tr))])   # 설계행렬 (n x 2)
y = ratio[tr]

# 정규방정식: (X^T X) theta = X^T y
XtX, Xty = X.T @ X, X.T @ y
theta_mine = np.linalg.solve(XtX, Xty)
theta_np   = np.linalg.lstsq(X, y, rcond=None)[0]

print("X^T X =\n", XtX)
print(f"\n직접 푼 해   : w={theta_mine[0]:+.6f}  b={theta_mine[1]:+.6f}")
print(f"lstsq의 해   : w={theta_np[0]:+.6f}  b={theta_np[1]:+.6f}")
print(f"두 해의 최대 차이: {np.abs(theta_mine - theta_np).max():.2e}")

# 손실이 정말 최소인지 주변을 흔들어 확인
def mse(t):
    return float(np.mean((X @ t - y) ** 2))
print(f"\nMSE(해)          = {mse(theta_mine):.8f}")
for d in (0.01, -0.01):
    print(f"MSE(w{d:+.2f})       = {mse(theta_mine + [d, 0]):.8f}")
    print(f"MSE(b{d:+.2f})       = {mse(theta_mine + [0, d]):.8f}")

#!/usr/bin/env python3
"""설계행렬에 열을 추가하면 같은 최소제곱이 어디까지 좋아지는지 비교한다."""
import numpy as np
from data import load

ext, size, entropy, ratio, tr, te = load()
logsize = np.log10(size)

FEATURES = {
    "entropy":                       lambda: [entropy],
    "entropy + log10(size)":         lambda: [entropy, logsize],
    "entropy + entropy^2":           lambda: [entropy, entropy ** 2],
    "entropy + entropy^2 + ^3":      lambda: [entropy, entropy ** 2, entropy ** 3],
    "위 3개 + log10(size)":          lambda: [entropy, entropy ** 2, entropy ** 3, logsize],
}

def design(cols, rows):
    return np.column_stack([c[rows] for c in cols] + [np.ones(len(rows))])

print(f"{'설계행렬 열 구성':<28}{'열 수':>5}{'학습 MAE':>11}{'평가 MAE':>11}{'평가 R^2':>10}")
print("-" * 66)
for name, build in FEATURES.items():
    cols = build()
    Xtr, Xte = design(cols, tr), design(cols, te)
    theta = np.linalg.lstsq(Xtr, ratio[tr], rcond=None)[0]
    pred_tr, pred_te = Xtr @ theta, Xte @ theta
    mae_tr = np.abs(pred_tr - ratio[tr]).mean()
    mae_te = np.abs(pred_te - ratio[te]).mean()
    ss_res = ((pred_te - ratio[te]) ** 2).sum()
    ss_tot = ((ratio[te] - ratio[te].mean()) ** 2).sum()
    print(f"{name:<24}{Xtr.shape[1]:>7}{mae_tr:>11.4f}{mae_te:>11.4f}{1 - ss_res / ss_tot:>10.4f}")

#!/usr/bin/env python3
"""정규방정식이 깨지는 경우와 lstsq가 버티는 이유를 확인한다."""
import numpy as np
from data import load

ext, size, entropy, ratio, tr, te = load()

def report(name, cols):
    X = np.column_stack(cols + [np.ones(len(tr))])
    XtX = X.T @ X
    print(f"[{name}]  cond(X^T X) = {np.linalg.cond(XtX):.3e}")
    try:
        theta = np.linalg.solve(XtX, X.T @ ratio[tr])
        print(f"    solve  -> {np.array2string(theta, precision=4, floatmode='fixed')}")
    except np.linalg.LinAlgError as e:
        print(f"    solve  -> LinAlgError: {e}")
    theta, *_ = np.linalg.lstsq(X, ratio[tr], rcond=None)
    print(f"    lstsq  -> {np.array2string(theta, precision=4, floatmode='fixed')}")
    print(f"    lstsq MSE = {np.mean((X @ theta - ratio[tr]) ** 2):.8f}")

e = entropy[tr]
report("정상: entropy", [e])
report("완전 중복: entropy를 두 번", [e, e.copy()])
report("거의 중복: entropy와 entropy+1e-9", [e, e + 1e-9])

#!/usr/bin/env python3
"""남은 오차가 어디에 몰려 있는지 확장자별로 본다."""
import numpy as np
from data import load

ext, size, entropy, ratio, tr, te = load()

def fit_predict(cols):
    Xtr = np.column_stack([c[tr] for c in cols] + [np.ones(len(tr))])
    X   = np.column_stack(list(cols) + [np.ones(len(ext))])
    theta = np.linalg.lstsq(Xtr, ratio[tr], rcond=None)[0]
    return X @ theta

pred1 = fit_predict([entropy])
pred2 = fit_predict([entropy, entropy ** 2])

groups = [(e, np.where(ext[te] == e)[0]) for e in sorted(set(ext[te]))]
groups = [(e, i) for e, i in groups if len(i) >= 6]

print(f"{'확장자':<12}{'개수':>5}{'평균 ratio':>12}{'MAE(1차)':>11}{'MAE(2차)':>11}")
print("-" * 51)
for e, i in sorted(groups, key=lambda g: -np.abs(pred1[te][g[1]] - ratio[te][g[1]]).mean()):
    r = ratio[te][i]
    m1 = np.abs(pred1[te][i] - r).mean()
    m2 = np.abs(pred2[te][i] - r).mean()
    print(f"{e:<12}{len(i):>5}{r.mean():>12.3f}{m1:>11.4f}{m2:>11.4f}")