tdpeuter/2025ML-project-neural_compression
tdpeuter/2025ML-project-neural_compression
Archived
1
Fork 0
Code Issues Pull requests Releases 1 Activity Actions

feat: graphs + models + updated finished graph code + data in csv

Browse source
This commit is contained in:
RobinMeersman 2025-12-16 10:06:47 +01:00
parent f3b07c1df3
commit 1abc2f5113
34 changed files with 76 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

18
measure.py
View file

@ -1,5 +1,4 @@
import os
from argparse import ArgumentParser
from contextlib import contextmanager
import torch
@ -54,13 +53,15 @@ if __name__ == "__main__":
]
files_enwik9 = [
# "text.txt",
# "txt_large.txt",
# "txt_xlarge.txt"
"text.txt",
"text_large.txt",
"text_xlarge.txt"
]
files_enwik9_cnn = [
"text_small.txt",
"text_xsmall.txt",
"text_xxsmall.txt"
]
models = [
@ -69,7 +70,7 @@ if __name__ == "__main__":
("cnn-genome-full-256.pt", 256, "cnn", files_genome_cnn),
("cnn-genome-full-128.pt", 128, "cnn", files_genome_cnn),
("auto-enwik9-full-256.pt", 256, "autoencoder", files_enwik9),
("auto-enwik9-full-128", 128, "autoencoder", files_enwik9),
("auto-enwik9-full-128.pt", 128, "autoencoder", files_enwik9),
("cnn-enwik9-full-256.pt", 256, "cnn", files_enwik9_cnn),
("cnn-enwik9-full-128.pt", 128, "cnn", files_enwik9_cnn),
]
@ -78,10 +79,11 @@ if __name__ == "__main__":
with open("./results/compress/compression_results.csv", "w") as f:
# write header
f.write(
"model_type,model_name,context_length,input_file_name,original_file_size,compressed_file_size,match_percentage,compression_time,decompression_time\n"
"model_type,model_name,context_length,dataset_type,input_file_name,original_file_size,compressed_file_size,match_percentage,compression_time,decompression_time\n"
)
for model, context_length, model_name, files in models:
dataset_type = "genome" if "genome" in model else "enwik9"
for file in files:
in_file = f"./data/compression_sets/{file}"
model_path = f"./models/{model_name}/{model}"
@ -119,5 +121,5 @@ if __name__ == "__main__":
os.remove("./output/tmp.pt")
f.write(
f"{model_name},{model},{context_length},{file},{og_file_len},{compressed_size},{accuracy},{compression_time},{decompression_time}\n"
f"{model_name},{model},{context_length},{dataset_type},{file},{og_file_len},{compressed_size},{accuracy},{compression_time},{decompression_time}\n"
)