feat: datasets package
This commit is contained in:
parent
6de4db24cc
commit
ed44d5b283
1 changed files with 11 additions and 0 deletions
11
CNN-model/datasets/EnWik9.py
Normal file
11
CNN-model/datasets/EnWik9.py
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
from datasets import load_dataset
|
||||||
|
from os.path import curdir, join
|
||||||
|
|
||||||
|
class EnWik9DataSet:
|
||||||
|
def __init__(self):
|
||||||
|
path = join(curdir, "data")
|
||||||
|
self.data = load_dataset("haukur/enwik9", cache_dir=path, split="train")
|
||||||
|
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.data)
|
||||||
Reference in a new issue