adding some constants for BERT (#113)

Co-authored-by: Xinyuan Huang <[email protected]>

mlperf_logging/mllog/constants.py

@@ -46,12 +46,14 @@
 TRANSFORMER = "transformer"
 RNNT = "rnnt"
 UNET3D = "unet3d"
+BERT ="bert"
 
 # Constant values - model info
 ADAM = "adam"
 LARS = "lars"
 LAZY_ADAM = "lazy_adam"
 SGD = "sgd"
+LAMB ="lamb"
 
 # Constant values - metadata info
 ABORTED = "aborted"
@@ -101,13 +103,15 @@
 MIN_IMAGE_SIZE = "min_image_size"
 MODEL_BN_SPAN = "model_bn_span"
 NUM_IMAGE_CANDIDATES = "num_image_candidates"
+NUM_WARMUP_STEPS = "num_warmup_steps"
 OPT_ADAM_BETA_1 = "opt_adam_beta_1"
 OPT_ADAM_BETA_2 = "opt_adam_beta_2"
 OPT_ADAM_EPSILON = "opt_adam_epsilon"
 OPT_NAME = "opt_name"
 OPT_BASE_LR = "opt_base_learning_rate"
 OPT_LAMB_LR_MIN = "opt_lamb_learning_rate_min"
 OPT_LAMB_LR_DECAY_POLY_POWER = "opt_lamb_learning_rate_decay_poly_power"
+OPT_LAMB_WEIGHT_DECAY = "opt_lamb_weight_decay_rate"
 OPT_LAMB_BETA_1 = "opt_lamb_beta_1"
 OPT_LAMB_BETA_2 = "opt_lamb_beta_2"
 OPT_LAMB_EPSILON = "opt_lamb_epsilon"
@@ -121,6 +125,7 @@
 OPT_LR_DECAY_START_STEP = "opt_learning_rate_decay_start_step"
 OPT_LR_DECAY_STEPS = "opt_learning_rate_decay_steps"
 OPT_LR_REMAIN_STEPS = "opt_learning_rate_remain_steps"
+OPT_LR_TRAINING_STEPS = "opt_learning_rate_training_steps"
 OPT_LR_WARMUP_EPOCHS = "opt_learning_rate_warmup_epochs"
 OPT_LR_WARMUP_FACTOR = "opt_learning_rate_warmup_factor"
 OPT_LR_WARMUP_STEPS = "opt_learning_rate_warmup_steps"
@@ -140,6 +145,7 @@
 MODEL_EVAL_EMA_FACTOR = "model_eval_ema_factor"
 MODEL_WEIGHTS_INITIALIZATION_SCALE = "model_weights_initialization_scale"
 EVAL_MAX_PREDICTION_SYMBOLS = "eval_max_prediction_symbols"
+START_WARMUP_STEP ="start_warmup_step"
 
 # Log keys - misc.
 BBOX = "bbox"