feat: Choose dataset with options
This commit is contained in:
parent
20bdd4f566
commit
81c767371e
5 changed files with 67 additions and 60 deletions
|
|
@ -1,21 +1,19 @@
|
|||
from typing import Callable
|
||||
|
||||
import torch
|
||||
from os.path import curdir, join
|
||||
from lorem.text import TextLorem
|
||||
|
||||
from .Dataset import Dataset
|
||||
|
||||
|
||||
class LoremIpsumDataset(Dataset):
|
||||
def __init__(self, root: str = "data", transform: Callable = None):
|
||||
super().__init__(root, transform)
|
||||
def __init__(self, root: str | None = None, transform: Callable = None, size: int = 512):
|
||||
super().__init__('lorem_ipsum', root, transform)
|
||||
|
||||
# Generate text and convert to bytes
|
||||
_lorem = TextLorem()
|
||||
_text = ' '.join(_lorem._word() for _ in range(512))
|
||||
_text = ' '.join(_lorem._word() for _ in range(size))
|
||||
|
||||
path = join(curdir, "data")
|
||||
self._root = path
|
||||
# Convert text to bytes (UTF-8 encoded)
|
||||
self.dataset = torch.tensor([ord(c) % 256 for c in list(_text)], dtype=torch.long)
|
||||
self.context_length = 128
|
||||
|
|
|
|||
Reference in a new issue