chore: Replace firefox with 7zip (smaller)

README.md

@@ -1,5 +1,11 @@
 # neural compression
 
+## Running locally
+
+```
+uv sync --all-extras
+```
+
 Example usage:
 
 ```shell
@@ -21,10 +27,12 @@ python benchmark.py --debug compress \
   --input-file inputfile --output-file outputfile
 ```
 
-## Running locally
+Testing compression:
 
-```
+```shell
-uv sync --all-extras
+bash config/download_datasets.sh config/urls.txt /home/tdpeuter/data/ml-inputs
+bash config/generate_csv.sh > config/sub.csv
+bash config/local.sh
 ```
 
 ## Running on the Ghent University HPC

config/local.sh

@@ -9,6 +9,8 @@ ID="${JOBID}-${GIT_HASH}-${DATE}"
 STAT_FILE="results/${ID}/results.csv"
 MODELS=/home/tdpeuter/data/ml-models
 
+mkdir -p "results/${ID}"
+
 while read -r line; do
   IFS=',' read -r id input model dataset context <<< "$line"
 
@@ -16,11 +18,20 @@ while read -r line; do
     continue
   fi
 
+  output="results/${ID}/$(basename "${input}").${id}.pt"
+
   python main.py compress \
     --model-load-path "${MODELS}/${dataset}/${context}/${model}-1024.pt" \
     --input-file "${input}" \
-    --output-file "results/${ID}/${input}.pt" &
+    --output-file "${output}"
+
+  in_bytes="$(stat -c %s -- "${input}")"
+  out_bytes="$(stat -c %s -- "${output}")"
+
+  printf "%d,%s,%s,%s,%d,%d,%d\n" "$id" "$input" "$model" "$dataset" "$context" "$in_bytes" "$out_bytes" >> "${STAT_FILE}"
+
   exit_code="${?}"
+
   if [ "${exit_code}" -eq 0 ]; then
     echo "DONE"
   fi

config/sub.csv

@@ -1,5 +1,9 @@
 id,input,model,dataset,context_size
-0,/home/tdpeuter/data/ml-inputs/Firefox Setup 146.0.exe,cnn,enwik9,64
+0,/home/tdpeuter/data/ml-inputs/7z2501-x64.exe,cnn,enwik9,64
-1,/home/tdpeuter/data/ml-inputs/Firefox Setup 146.0.exe,cnn,human_reference,64
+1,/home/tdpeuter/data/ml-inputs/7z2501-x64.exe,cnn,human_reference,64
-2,/home/tdpeuter/data/ml-inputs/GCF_000005845.2_ASM584v2_genomic.fna,cnn,enwik9,64
+2,/home/tdpeuter/data/ml-inputs/Firefox Setup 146.0.exe,cnn,enwik9,64
-3,/home/tdpeuter/data/ml-inputs/GCF_000005845.2_ASM584v2_genomic.fna,cnn,human_reference,64
+3,/home/tdpeuter/data/ml-inputs/Firefox Setup 146.0.exe,cnn,human_reference,64
+4,/home/tdpeuter/data/ml-inputs/GCF_000005845.2_ASM584v2_genomic.fna,cnn,enwik9,64
+5,/home/tdpeuter/data/ml-inputs/GCF_000005845.2_ASM584v2_genomic.fna,cnn,human_reference,64
+6,/home/tdpeuter/data/ml-inputs/GCF_000005845.2_ASM584v2_genomic.fna.gz,cnn,enwik9,64
+7,/home/tdpeuter/data/ml-inputs/GCF_000005845.2_ASM584v2_genomic.fna.gz,cnn,human_reference,64

config/urls.txt

@@ -1,2 +1,2 @@
-https://download.mozilla.org/?product=firefox-latest&os=win&lang=en-US
 https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_ASM584v2_genomic.fna.gz
+https://www.7-zip.org/a/7z2501-x64.exe