feat: i think i set up the encoder
This commit is contained in:
parent
947aba31ee
commit
63d1b6f5ae
2 changed files with 51 additions and 6 deletions
|
|
@ -3,7 +3,7 @@ from torch.utils.data import TensorDataset
|
|||
|
||||
|
||||
def make_context_pairs(data: bytes, context_length: int) -> TensorDataset:
|
||||
data = torch.tensor(data, dtype=torch.uint8)
|
||||
data = torch.tensor(list(data), dtype=torch.uint8)
|
||||
sample_count = data.shape[0] - context_length
|
||||
x = data.unfold(0, context_length, 1)[:sample_count]
|
||||
y = data[context_length:]
|
||||
|
|
|
|||
Reference in a new issue