55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
from argparse import ArgumentParser
|
|
from math import ceil
|
|
|
|
import torch
|
|
from torch.utils.data import DataLoader, TensorDataset
|
|
|
|
from datasets import EnWik9DataSet, LoremIpsumDataset
|
|
from trainers import OptunaTrainer, Trainer
|
|
|
|
BATCH_SIZE = 64
|
|
DEVICE = torch.accelerator.current_accelerator().type if torch.accelerator.is_available() else "cpu"
|
|
|
|
# hyper parameters
|
|
context_length = 128
|
|
|
|
if __name__ == "__main__":
|
|
print(f"Running on device: {DEVICE}...")
|
|
parser = ArgumentParser()
|
|
parser.add_argument("--method", choices=["optuna", "train"], required=True)
|
|
parser.add_argument("--model-path", type=str, required=False)
|
|
args = parser.parse_args()
|
|
|
|
if args.method == "train":
|
|
dataset = EnWik9DataSet()
|
|
elif args.method == "optuna":
|
|
dataset = LoremIpsumDataset()
|
|
else:
|
|
raise ValueError(f"Unknown method: {args.method}")
|
|
|
|
dataset_length = len(dataset)
|
|
training_size = ceil(0.8 * dataset_length)
|
|
|
|
print(f"training set size = {training_size}, validation set size {dataset_length - training_size}")
|
|
data = dataset.data["text"]
|
|
|
|
train_set, validate_set = torch.utils.data.random_split(TensorDataset(data),
|
|
[training_size, dataset_length - training_size])
|
|
training_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
|
|
validation_loader = DataLoader(validate_set, batch_size=BATCH_SIZE, shuffle=False)
|
|
loss_fn = torch.nn.CrossEntropyLoss()
|
|
|
|
model = None
|
|
if args.model_path is not None:
|
|
model = torch.load(args.model_path)
|
|
|
|
trainer: Trainer = OptunaTrainer() if args.method == "optuna" else None
|
|
|
|
trainer.execute(
|
|
model=model,
|
|
train_loader=training_loader,
|
|
validation_loader=validation_loader,
|
|
loss_fn=loss_fn,
|
|
n_epochs=200,
|
|
device=DEVICE
|
|
)
|