Slightly more correct metadata for quantized config.json

Author: turboderp

conversion/tokenize.py

@@ -48,6 +48,10 @@ def tokenize(job, save_fn, tokenizer, measure = False):
         cal_tokens = get_tokens(rows, length, cal_ds, tokenizer)
     else:
         cal_tokens = get_standard_calibration(job, measure, tokenizer)
+        if measure:
+            job["measurement_rows"] = cal_tokens.shape[0]
+        else:
+            job["dataset_rows"] = cal_tokens.shape[0]
 
     cal_filename = os.path.join(job["out_dir"], "cal_data.safetensors")
     cal_dict = { "input_ids": cal_tokens }