Skip to content

Commit

Permalink
better evaluation
Browse files Browse the repository at this point in the history
  • Loading branch information
@JanProvaznik
JanProvaznik committed Mar 18, 2024
1 parent 1949ab9 commit 4048c05
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 18 deletions.
65 changes: 48 additions & 17 deletions evaluation_batchedgpuevaluate_other_models.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,31 +107,62 @@
" 'en_noisevignere3_4000': Model(ByT5NoisyVignere3Dataset, 'en_noisevignere3_4000', 'en', 22819, True, 4000, .15), # 22900\n",
"\n",
" # de 22904\n",
" 'de_noisevignere3_500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_500', 'de', 22904 , True, 500, .15), # \n",
" 'de_noisevignere3_1000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_1000', 'de', 22904 , True, 1000, .15), # \n",
" 'de_noisevignere3_1500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_1500', 'de', 22904 , True, 1500, .15), #\n",
" 'de_noisevignere3_2000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_2000', 'de', 22904 , True, 2000, .15), #\n",
" 'de_noisevignere3_2500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_2500', 'de', 22904 , True, 2500, .15), #\n",
" 'de_noisevignere3_3000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_3000', 'de', 22904 , True, 3000, .15), #\n",
" 'de_noisevignere3_3500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_3500', 'de', 22904 , True, 3500, .15), #\n",
" 'de_noisevignere3_4000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_4000', 'de', 22904 , True, 4000, .15), #\n",
" 'de_noisevignere3_500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_500', 'de', 22904 , True, 500, .15), # 23104\n",
" 'de_noisevignere3_1000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_1000', 'de', 22904 , True, 1000, .15), # 23105\n",
" 'de_noisevignere3_1500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_1500', 'de', 22904 , True, 1500, .15), # 23150\n",
" 'de_noisevignere3_2000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_2000', 'de', 22904 , True, 2000, .15), # 23161\n",
" 'de_noisevignere3_2500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_2500', 'de', 22904 , True, 2500, .15), # 23162\n",
" 'de_noisevignere3_3000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_3000', 'de', 22904 , True, 3000, .15), # 23163\n",
" 'de_noisevignere3_3500': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_3500', 'de', 22904 , True, 3500, .15), # 23164\n",
" 'de_noisevignere3_4000': Model(ByT5NoisyVignere3Dataset, 'de_noisevignere3_4000', 'de', 22904 , True, 4000, .15), # 23165\n",
"\n",
" # cs 22989\n",
" 'cs_noisevignere3_500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_500', 'cs', 22989 , True, 500, .15), #\n",
" 'cs_noisevignere3_1000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_1000', 'cs', 22989 , True, 1000, .15), #\n",
" 'cs_noisevignere3_1500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_1500', 'cs', 22989 , True, 1500, .15), #\n",
" 'cs_noisevignere3_2000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_2000', 'cs', 22989 , True, 2000, .15), #\n",
" 'cs_noisevignere3_2500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_2500', 'cs', 22989 , True, 2500, .15), #\n",
" 'cs_noisevignere3_3000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_3000', 'cs', 22989 , True, 3000, .15), #\n",
" 'cs_noisevignere3_3500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_3500', 'cs', 22989 , True, 3500, .15), #\n",
" 'cs_noisevignere3_4000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_4000', 'cs', 22989 , True, 4000, .15), #\n",
" 'cs_noisevignere3_500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_500', 'cs', 22989 , True, 500, .15), # 23166\n",
" 'cs_noisevignere3_1000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_1000', 'cs', 22989 , True, 1000, .15), # 23168\n",
" 'cs_noisevignere3_1500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_1500', 'cs', 22989 , True, 1500, .15), # 23169\n",
" 'cs_noisevignere3_2000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_2000', 'cs', 22989 , True, 2000, .15), # 23170\n",
" 'cs_noisevignere3_2500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_2500', 'cs', 22989 , True, 2500, .15), # 23171\n",
" 'cs_noisevignere3_3000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_3000', 'cs', 22989 , True, 3000, .15), # 23177\n",
" 'cs_noisevignere3_3500': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_3500', 'cs', 22989 , True, 3500, .15), # 23178\n",
" 'cs_noisevignere3_4000': Model(ByT5NoisyVignere3Dataset, 'cs_noisevignere3_4000', 'cs', 22989 , True, 4000, .15), # 23179\n",
"\n",
" # de enigma 23190\n",
" 'de_noiseconstenigma_500': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_500', 'de', 23190 , True, 500, .15), # \n",
" 'de_noiseconstenigma_1000': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_1000', 'de', 23190 , True, 1000, .15), #\n",
" 'de_noiseconstenigma_1500': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_1500', 'de', 23190 , True, 1500, .15), #\n",
" 'de_noiseconstenigma_2000': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_2000', 'de', 23190 , True, 2000, .15), #\n",
" 'de_noiseconstenigma_2500': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_2500', 'de', 23190 , True, 2500, .15), #\n",
" 'de_noiseconstenigma_3000': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_3000', 'de', 23190 , True, 3000, .15), #\n",
" 'de_noiseconstenigma_3500': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_3500', 'de', 23190 , True, 3500, .15), #\n",
" 'de_noiseconstenigma_4000': Model(ByT5NoisyConstEnigmaDataset, 'de_noiseconstenigma_4000', 'de', 23190 , True, 4000, .15), #\n",
"\n",
"\n",
"\n",
" # cs enigma 23167 \n",
" 'cs_noiseconstenigma_500': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_500', 'cs', 23167 , True, 500, .15), #\n",
" 'cs_noiseconstenigma_1000': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_1000', 'cs', 23167 , True, 1000, .15), #\n",
" 'cs_noiseconstenigma_1500': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_1500', 'cs', 23167 , True, 1500, .15), #\n",
" 'cs_noiseconstenigma_2000': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_2000', 'cs', 23167 , True, 2000, .15), #\n",
" 'cs_noiseconstenigma_2500': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_2500', 'cs', 23167 , True, 2500, .15), #\n",
" 'cs_noiseconstenigma_3000': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_3000', 'cs', 23167 , True, 3000, .15), #\n",
" 'cs_noiseconstenigma_3500': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_3500', 'cs', 23167 , True, 3500, .15), #\n",
" 'cs_noiseconstenigma_4000': Model(ByT5NoisyConstEnigmaDataset, 'cs_noiseconstenigma_4000', 'cs', 23167 , True, 4000, .15), #\n",
"\n",
" # en enigma 23609\n",
" 'en_noiseconstenigma_500': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_500', 'en', 23609 , True, 500, .15), #\n",
" 'en_noiseconstenigma_1000': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_1000', 'en', 23609 , True, 1000, .15), #\n",
" 'en_noiseconstenigma_1500': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_1500', 'en', 23609 , True, 1500, .15), #\n",
" 'en_noiseconstenigma_2000': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_2000', 'en', 23609 , True, 2000, .15), #\n",
" 'en_noiseconstenigma_2500': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_2500', 'en', 23609 , True, 2500, .15), #\n",
" 'en_noiseconstenigma_3000': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_3000', 'en', 23609 , True, 3000, .15), #\n",
" 'en_noiseconstenigma_3500': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_3500', 'en', 23609 , True, 3500, .15), #\n",
" 'en_noiseconstenigma_4000': Model(ByT5NoisyConstEnigmaDataset, 'en_noiseconstenigma_4000', 'en', 23609 , True, 4000, .15), #\n",
" \n",
"\n",
"}\n",
"\n",
"# evaluated_name = 'en_noisevignere_checkpoint-5000'\n",
"evaluated_name = 'de_noisevignere3_500'\n",
"evaluated_name = 'cs_noisevignere3_4000'\n",
"model_metadata = models[evaluated_name]\n",
"\n",
"data_path = f'news.2013.{model_metadata.language}.trainlen.200.evaluation.100000.csv'\n",
Expand Down
4 changes: 3 additions & 1 deletion run_notebook.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ jupyter nbconvert --to python "$name"

script_name=script_"$SLURM_JOB_ID".py
mv "$name_without_extension".py "$logdir"/"$script_name"
python "$logdir"/"$script_name"

echo "$@"
python "$logdir"/"$script_name" "$@"


# copy the script to the log directory, with appended slurm job id
Expand Down

0 comments on commit 4048c05

Please sign in to comment.