feat: uhm, i changed some things

This commit is contained in:
RobinMeersman 2025-11-25 20:20:08 +01:00
parent b58682cb49
commit 6de4db24cc
27 changed files with 1302 additions and 137 deletions

View file

@ -1,20 +0,0 @@
import torch
from torch.utils.data import TensorDataset
import matplotlib.pyplot as plt
def make_context_pairs(data: bytes, context_length: int) -> TensorDataset:
data = torch.tensor(list(data), dtype=torch.long)
sample_count = data.shape[0] - context_length
x = data.unfold(0, context_length, 1)[:sample_count]
y = data[context_length:]
return TensorDataset(x, y)
def print_distribution(from_to: tuple[int, int], probabilities: list[float]):
plt.hist(range(from_to[0], from_to[1]), weights=probabilities)
plt.show()
def load_data(path: str) -> bytes:
with open(path, "rb") as f:
return f.read()