feat: Choose dataset with options

This commit is contained in:
Tibo De Peuter 2025-11-30 20:19:39 +01:00
parent 20bdd4f566
commit 81c767371e
Signed by: tdpeuter
GPG key ID: 38297DE43F75FFE2
5 changed files with 67 additions and 60 deletions

View file

@ -1,21 +1,19 @@
from typing import Callable
import torch
from os.path import curdir, join
from lorem.text import TextLorem
from .Dataset import Dataset
class LoremIpsumDataset(Dataset):
def __init__(self, root: str = "data", transform: Callable = None):
super().__init__(root, transform)
def __init__(self, root: str | None = None, transform: Callable = None, size: int = 512):
super().__init__('lorem_ipsum', root, transform)
# Generate text and convert to bytes
_lorem = TextLorem()
_text = ' '.join(_lorem._word() for _ in range(512))
_text = ' '.join(_lorem._word() for _ in range(size))
path = join(curdir, "data")
self._root = path
# Convert text to bytes (UTF-8 encoded)
self.dataset = torch.tensor([ord(c) % 256 for c in list(_text)], dtype=torch.long)
self.context_length = 128