Solving mixer ties with runtime as second criteria

lightwood/api/json_ai.py

@@ -483,6 +483,7 @@ def add_implicit_values(json_ai: JsonAI) -> JsonAI:
         'fit': json_ai.model["args"].get("fit", True),
         'args': json_ai.model["args"].get("args", "$pred_args"),
         'accuracy_functions': json_ai.model["args"].get("accuracy_functions", "$accuracy_functions"),
+        'runtime_log': json_ai.model["args"].get("runtime_log", "$runtime_log"),
         'ts_analysis': json_ai.model["args"].get("ts_analysis", "self.ts_analysis" if is_ts else None),
         'dtype_dict': json_ai.model["args"].get("dtype_dict", "$dtype_dict"),
     }

lightwood/ensemble/best_of.py

@@ -23,7 +23,8 @@ class BestOf(BaseEnsemble):
     indexes_by_accuracy: List[float]
 
     def __init__(self, target, mixers: List[BaseMixer], data: EncodedDs, accuracy_functions,
-                 args: PredictionArguments, ts_analysis: Optional[dict] = None, fit: bool = True) -> None:
+                 args: PredictionArguments, runtime_log: Optional[dict] = None,
+                 ts_analysis: Optional[dict] = None, fit: bool = True) -> None:
         super().__init__(target, mixers, data, fit=False)
         self.special_dispatch_list = [GluonTSMixer]
 
@@ -55,9 +56,17 @@ def __init__(self, target, mixers: List[BaseMixer], data: EncodedDs, accuracy_fu
                     is_best=False
                 ))
 
-                score_list.append(avg_score)
+                if runtime_log is not None:
+                    score_list.append((avg_score, -1 * runtime_log[type(mixer).__name__ + '.fit_mixer'][0][0]))
+                else:
+                    score_list.append(avg_score)
 
-            self.indexes_by_accuracy = list(reversed(np.array(score_list).argsort()))
+            if runtime_log is not None:
+                self.indexes_by_accuracy = list(reversed(np.array(score_list,
+                                                dtype=[('x', np.float32), ('y', np.float32)]).argsort(
+                                                    order=('x', 'y'))))
+            else:
+                self.indexes_by_accuracy = list(reversed(np.array(score_list).argsort()))
             self.supports_proba = self.mixers[self.indexes_by_accuracy[0]].supports_proba
             log.info(f'Picked best mixer: {type(self.mixers[self.indexes_by_accuracy[0]]).__name__}')
             self.prepared = True

lightwood/helpers/log.py

@@ -30,8 +30,12 @@ def wrap(predictor, *args, **kw):
         result = f(predictor, *args, **kw)
         te = time()
         log.debug(f' `{f.__name__}` runtime: {round(te - ts, 2)} seconds')
+        name_ = f.__name__ if f.__name__ != 'fit_mixer' else type(args[0]).__name__ + '.fit_mixer'
         if hasattr(predictor, 'runtime_log'):
-            predictor.runtime_log[(f.__name__, datetime.fromtimestamp(ts))] = round(te - ts, 2)
+            if name_ not in predictor.runtime_log:
+                predictor.runtime_log[name_] = [(round(te - ts, 2), datetime.fromtimestamp(ts))]
+            else:
+                predictor.runtime_log[name_].append((round(te - ts, 2), datetime.fromtimestamp(ts)))
         return result
     return wrap