От чисел к тензорам: библиотека nanotorch

import numpy as np

rng = np.random.default_rng(0)
A, B = rng.normal(size=(3, 4)), rng.normal(size=(4, 2))
C = rng.normal(size=(3, 2))
L = lambda A, B: ((A @ B) * C).sum()     # тогда ∂L/∂Y = C

G = C
dA, dB = G @ B.T, A.T @ G

h = 1e-6
E = np.zeros_like(A); E[1, 2] = h
print("∂L/∂A[1,2]: формула", round(dA[1, 2], 6), " численно", round((L(A + E, B) - L(A - E, B)) / (2 * h), 6))
E = np.zeros_like(B); E[3, 0] = h
print("∂L/∂B[3,0]: формула", round(dB[3, 0], 6), " численно", round((L(A, B + E) - L(A, B - E)) / (2 * h), 6))

import numpy as np

def unbroadcast(grad, shape):
    while grad.ndim > len(shape):
        grad = grad.sum(axis=0)
    for axis, size in enumerate(shape):
        if size == 1 and grad.shape[axis] != 1:
            grad = grad.sum(axis=axis, keepdims=True)
    return grad

class Tensor:
    def __init__(self, data, _children=()):
        self.data = np.asarray(data, dtype=float)
        self.grad = np.zeros_like(self.data)
        self._backward = lambda: None
        self._prev = _children

    def __add__(self, other):
        out = Tensor(self.data + other.data, (self, other))
        def _backward():
            self.grad += unbroadcast(out.grad, self.data.shape)
            other.grad += unbroadcast(out.grad, other.data.shape)
        out._backward = _backward
        return out

    def __matmul__(self, other):
        out = Tensor(self.data @ other.data, (self, other))
        def _backward():
            self.grad += out.grad @ other.data.T
            other.grad += self.data.T @ out.grad
        out._backward = _backward
        return out

    def mse(self, target):
        diff = self.data - target
        out = Tensor((diff ** 2).mean(), (self,))
        def _backward():
            self.grad += 2 * diff / diff.size * out.grad
        out._backward = _backward
        return out

    def backward(self):
        topo, seen = [], set()
        def build(t):
            if id(t) not in seen:
                seen.add(id(t))
                for c in t._prev:
                    build(c)
                topo.append(t)
        build(self)
        self.grad = np.ones_like(self.data)
        for t in reversed(topo):
            t._backward()

rng = np.random.default_rng(0)
X = rng.normal(size=(100, 3))
y = X @ np.array([[2.0], [-1.0], [0.5]]) + 3

W, b = Tensor(np.zeros((3, 1))), Tensor(np.zeros(1))
for step in range(200):
    loss = (Tensor(X) @ W + b).mse(y)
    W.grad[:] = 0; b.grad[:] = 0
    loss.backward()
    W.data -= 0.1 * W.grad
    b.data -= 0.1 * b.grad
print("W =", W.data.ravel().round(3), " b =", b.data.round(3), " loss =", round(float(loss.data), 6))

import inspect
import nanotorch as nt

print(inspect.getsource(nt.Tensor.__matmul__))
print(inspect.getsource(nt.Tensor.relu))

import numpy as np
import nanotorch as nt

x = nt.tensor([[1.0, 2.0], [3.0, 4.0]])                 # данные: градиент не нужен
w = nt.tensor([[0.5], [-1.0]], requires_grad=True)      # параметр

y = (x @ w).relu().sum()
y.backward()
print("y =", y.item())
print("∂y/∂w =", w.grad.ravel())
print("x.grad =", x.grad)                               # None — за x не следили

with nt.no_grad():                                      # граф не строится
    z = x @ w
print("z требует градиент:", z.requires_grad)

import numpy as np
import nanotorch as nt

rng = np.random.default_rng(1)
A = nt.tensor(rng.normal(size=(4, 3)), requires_grad=True)

def f(a):
    return ((a.tanh() @ a.T).softmax(axis=1) * 2).log().mean()

out = f(A)
out.backward()
num = nt.numerical_grad(lambda arr: f(nt.tensor(arr)).item(), A.data)
print("максимальная разница с численным градиентом:", float(np.abs(A.grad - num).max()))

import numpy as np
import nanotorch as nt

class MyLinear(nt.Module):
    def __init__(self, n_in, n_out):
        self.W = nt.Parameter(np.random.default_rng(0).normal(0, 0.1, size=(n_in, n_out)))
        self.b = nt.Parameter(np.zeros(n_out))

    def forward(self, x):
        return x @ self.W + self.b

class TwoLayer(nt.Module):
    def __init__(self):
        self.l1 = MyLinear(2, 16)
        self.l2 = MyLinear(16, 1)

    def forward(self, x):
        return self.l2(self.l1(x).relu())

model = TwoLayer()
print("параметров:", model.num_params())
for name, p in model.named_parameters():
    print(f"  {name:6} {p.shape}")
out = model(nt.tensor(np.ones((5, 2))))
print("выход:", out.shape)

import numpy as np
import nanotorch as nt
import vis

nt.manual_seed(0)
X, y = nt.make_moons(300, noise=0.15, seed=0)

model = nt.Sequential(nt.Linear(2, 32), nt.ReLU(), nt.Linear(32, 32), nt.ReLU(), nt.Linear(32, 2))
optimizer = nt.SGD(model.parameters(), lr=0.1)

for epoch in range(300):
    logits = model(nt.Tensor(X))               # 1. прямой проход
    loss = nt.cross_entropy(logits, y)         # 2. функция потерь
    optimizer.zero_grad()                      # 3. обнулить градиенты
    loss.backward()                            # 4. обратный проход
    optimizer.step()                           # 5. шаг оптимизатора
    if epoch % 75 == 0 or epoch == 299:
        print(f"эпоха {epoch:3}: loss = {loss.item():.4f}, точность = {nt.accuracy(logits, y):.1%}")

with nt.no_grad():
    predict = lambda pts: model(nt.Tensor(pts)).data.argmax(axis=1)
    vis.boundary(predict, X, y, title="Сеть на nanotorch: 2 → 32 → 32 → 2")