diff --git a/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb b/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb index 301c800..46bbca8 100644 --- a/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb +++ b/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb @@ -114,7 +114,7 @@ "id": "83701c19", "metadata": {}, "source": [ - "# For LSTM model" + "# For LSTM and BERT model" ] }, { @@ -125,68 +125,23 @@ "outputs": [], "source": [ "for meta_model in meta_model_names:\n", - " vocab_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-vocab.json')\n", - " merges_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-merges.txt')\n", - " tokenizer = TokenizerWrapperBPE(ByteLevelBPETokenizer(vocab=vocab_file,\n", - " merges=merges_file,\n", - " lowercase=True))\n", - " # load and sort out the config\n", - " config_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config.json\")\n", - " with open(config_file, 'r') as jfile:\n", - " config_dict = json.load(jfile)\n", - " config = ConfigMetaCAT()\n", - " for key, value in config_dict.items():\n", - " setattr(config, key, value['py/state']['__dict__'])\n", - " \n", + " \n", + " # load the meta_model\n", + " mc = MetaCAT.load(save_dir_path=os.path.join(base_dir_meta_models,\"meta_\"+meta_model))\n", + "\n", + " # changing parameters\n", + " mc.config.train['nepochs'] = 15\n", + "\n", " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", " #Ideally this should replace the meta_models inside the modelpack\n", "\n", - " # Initialise and train meta_model\n", - " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", + " # train the meta_model\n", " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", " \n", " # Save results\n", " json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results.json'), 'w'))" ] }, - { - "cell_type": "markdown", - "id": "91ff4e28", - "metadata": {}, - "source": [ - "# For BERT model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e255dda2", - "metadata": {}, - "outputs": [], - "source": [ - "for meta_model in meta_model_names:\n", - " # load and sort out the config\n", - " config_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config.json\")\n", - " with open(config_file, 'r') as jfile:\n", - " config_dict = json.load(jfile)\n", - " config = ConfigMetaCAT()\n", - " for key, value in config_dict.items():\n", - " setattr(config, key, value['py/state']['__dict__'])\n", - "\n", - " tokenizer = TokenizerWrapperBERT.load(os.path.join(base_dir_meta_models,\"meta_\"+meta_model), \n", - " config.model['model_variant'])\n", - " \n", - " # change model name if training BERT for the first time\n", - " config.model['model_name'] = 'bert'\n", - " \n", - " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", - " #Ideally this should replace the meta_models inside the modelpack\n", - "\n", - " # Initialise and train meta_model\n", - " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" - ] - }, { "cell_type": "markdown", "id": "ab23e424", diff --git a/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb b/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb index 7525068..7266056 100644 --- a/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb +++ b/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb @@ -34,7 +34,8 @@ "source": [ "# if you want to enable info level logging\n", "import logging\n", - "logging.basicConfig(level=logging.INFO,force=True)" + "logging.basicConfig(level=logging.INFO,force=True)\n", + "logger = logging.getLogger(__name__)" ] }, { @@ -96,1029 +97,47 @@ }, { "cell_type": "markdown", - "id": "83701c19", + "id": "699be74b", "metadata": {}, "source": [ - "# For LSTM model" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "f5a71c31", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:medcat.meta_cat:LSTM model used for classification\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {1: 75, 0: 75, 2: 75}\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 1684\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.46 0.40 0.43 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.92 0.94 0.93 6331\n", - "\n", - " accuracy 0.87 7188\n", - " macro avg 0.46 0.45 0.45 7188\n", - "weighted avg 0.86 0.87 0.87 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.68 0.67 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.95 0.96 0.95 1577\n", - "\n", - " accuracy 0.92 1797\n", - " macro avg 0.54 0.55 0.54 1797\n", - "weighted avg 0.91 0.92 0.92 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 0 and macro avg/f1-score: 0.5424475654733077 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.80 0.73 793\n", - " 1 0.61 0.17 0.27 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.64 0.65 7188\n", - "weighted avg 0.93 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.76 0.72 209\n", - " 1 0.36 0.36 0.36 11\n", - " 2 0.96 0.95 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.67 0.69 0.68 1797\n", - "weighted avg 0.93 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 1 and macro avg/f1-score: 0.6809929509005487 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.73 0.88 0.79 793\n", - " 1 0.47 0.44 0.46 64\n", - " 2 0.98 0.95 0.97 6331\n", - "\n", - " accuracy 0.94 7188\n", - " macro avg 0.73 0.76 0.74 7188\n", - "weighted avg 0.95 0.94 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.81 0.76 209\n", - " 1 0.31 0.45 0.37 11\n", - " 2 0.97 0.95 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.67 0.74 0.70 1797\n", - "weighted avg 0.94 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 2 and macro avg/f1-score: 0.6964827596275013 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.91 0.85 793\n", - " 1 0.62 0.73 0.67 64\n", - " 2 0.99 0.97 0.98 6331\n", - "\n", - " accuracy 0.96 7188\n", - " macro avg 0.80 0.87 0.83 7188\n", - "weighted avg 0.96 0.96 0.96 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.75 0.78 0.76 209\n", - " 1 0.31 0.36 0.33 11\n", - " 2 0.97 0.96 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.68 0.70 0.69 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.84 0.92 0.88 793\n", - " 1 0.73 0.91 0.81 64\n", - " 2 0.99 0.98 0.98 6331\n", - "\n", - " accuracy 0.97 7188\n", - " macro avg 0.86 0.94 0.89 7188\n", - "weighted avg 0.97 0.97 0.97 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.75 0.78 0.77 209\n", - " 1 0.27 0.36 0.31 11\n", - " 2 0.97 0.96 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.66 0.70 0.68 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.88 0.94 0.91 793\n", - " 1 0.81 0.92 0.86 64\n", - " 2 0.99 0.98 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.89 0.95 0.92 7188\n", - "weighted avg 0.98 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.76 0.76 209\n", - " 1 0.29 0.36 0.32 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.67 0.69 0.68 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.89 0.94 0.91 793\n", - " 1 0.85 0.95 0.90 64\n", - " 2 0.99 0.98 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.91 0.96 0.93 7188\n", - "weighted avg 0.98 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.76 0.78 209\n", - " 1 0.44 0.36 0.40 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.74 0.70 0.72 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 6 and macro avg/f1-score: 0.7163631972056036 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.90 0.94 0.92 793\n", - " 1 0.91 0.97 0.94 64\n", - " 2 0.99 0.99 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.94 0.97 0.95 7188\n", - "weighted avg 0.98 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.76 0.78 209\n", - " 1 0.44 0.36 0.40 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.74 0.70 0.72 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.91 0.96 0.94 793\n", - " 1 0.94 0.98 0.96 64\n", - " 2 0.99 0.99 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.95 0.98 0.96 7188\n", - "weighted avg 0.99 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.78 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.72 0.70 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.92 0.96 0.94 793\n", - " 1 0.93 0.98 0.95 64\n", - " 2 0.99 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.95 0.98 0.96 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.81 0.79 209\n", - " 1 0.38 0.45 0.42 11\n", - " 2 0.97 0.96 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.71 0.74 0.73 1797\n", - "weighted avg 0.95 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 9 and macro avg/f1-score: 0.7253847166674895 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.92 0.97 0.94 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.97 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.78 0.79 0.79 209\n", - " 1 0.33 0.45 0.38 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.69 0.74 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.92 0.97 0.94 793\n", - " 1 0.94 1.00 0.97 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.95 0.99 0.97 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.77 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.72 0.70 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.94 0.97 0.95 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.97 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.76 0.79 209\n", - " 1 0.36 0.36 0.36 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.95 1797\n", - " macro avg 0.71 0.70 0.71 1797\n", - "weighted avg 0.94 0.95 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.94 0.98 0.96 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.98 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.82 0.74 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.96 0.98 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.73 0.69 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.93 0.98 0.95 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.98 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.75 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.96 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.72 0.69 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n" - ] - } - ], - "source": [ - "for meta_model in meta_model_names:\n", - " vocab_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-vocab.json')\n", - " merges_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-merges.txt')\n", - " tokenizer = TokenizerWrapperBPE(ByteLevelBPETokenizer(vocab=vocab_file,\n", - " merges=merges_file,\n", - " lowercase=True))\n", - " # load and sort out the config\n", - " config_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config.json\")\n", - " with open(config_file, 'r') as jfile:\n", - " config_dict = json.load(jfile)\n", - " config = ConfigMetaCAT()\n", - " for key, value in config_dict.items():\n", - " setattr(config, key, value['py/state']['__dict__'])\n", - " \n", - " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", - " #Ideally this should replace the meta_models inside the modelpack\n", - " \n", - " #Below are the config values used for Experiencer classification task\n", - " \n", - " #Class weights--------------------------------------------------------------\n", - " #adjusting class weights to give more importance to minority classes\n", - " # To use class weights, we have 2 options:\n", - "\n", - " #1st option:\n", - " #to calculate class weights based on class distribution\n", - " #NOTE: this will only be applicable if config.train.class_weights is empty\n", - " config.train['class_weights'] = []\n", - " config.train['compute_class_weights'] = True\n", - "\n", - " #2nd option\n", - " #using specified class weights\n", - " config.train['class_weights'] = [0.4,1.5,0.1]\n", - "\n", - " #we'll use the 2nd option in this example\n", - " #----------------------------------------------------------------------------\n", - " \n", - " #NOTE: when using class weights, it is recommended to define the category to index mapping to ensure the weights are assigned to the right class\n", - " config.general['category_value2id'] = {'Family':1, 'Other':0, 'Patient':2}\n", - "\n", - " config.train['test_size'] = 0.2\n", - " config.train['nepochs'] = 15\n", - "\n", - " #since we have class imbalance, macro avg is better suited than weighted avg\n", - " config.train.metric['base'] = 'macro avg'\n", - "\n", - " # Initialise and train meta_model\n", - " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", - "\n", - " # Save results\n", - " json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results.json'), 'w'))" + "# Class weights " ] }, { "cell_type": "markdown", - "id": "0010d730", + "id": "e624d876", "metadata": {}, "source": [ - "LSTM has high weighted F1-score (0.94), however due to the class imbalance, that can be misleading.
The recall values for the minority classes (Other and Family) are low, especially for Family - 0.36\n", - "
There is room for improvement in performance, especially for the minority classes" + "Adjusting class weights to give more importance to specific classes. Generally, class weights are used in favour of minority classes(classes with less number of samples) to boost their performance.\n", + "

To use class weights, we have 2 options:\n", + "
1. calculate class weights based on class distribution\n", + "
2. using specified class weights" ] }, { "cell_type": "markdown", - "id": "91ff4e28", - "metadata": {}, - "source": [ - "# For BERT model" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "521479e4", + "id": "dc91f7d6", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", - "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", - "WARNING:medcat.utils.meta_cat.models:\n", - "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", - "INFO:medcat.meta_cat:BERT model used for classification\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {1: 75, 0: 75, 2: 75}\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 1078\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.12 0.25 0.16 793\n", - " 1 0.00 0.03 0.01 64\n", - " 2 0.88 0.68 0.77 6331\n", - "\n", - " accuracy 0.63 7188\n", - " macro avg 0.34 0.32 0.31 7188\n", - "weighted avg 0.79 0.63 0.70 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.00 0.00 0.00 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.88 1.00 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.29 0.33 0.31 1797\n", - "weighted avg 0.77 0.88 0.82 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 0 and macro avg/f1-score: 0.31159849832049 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.14 0.03 0.05 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.88 0.98 0.93 6331\n", - "\n", - " accuracy 0.86 7188\n", - " macro avg 0.34 0.34 0.33 7188\n", - "weighted avg 0.79 0.86 0.82 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.00 0.00 0.00 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.88 1.00 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.29 0.33 0.31 1797\n", - "weighted avg 0.77 0.88 0.82 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.26 0.06 0.10 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.88 0.98 0.93 6331\n", - "\n", - " accuracy 0.87 7188\n", - " macro avg 0.38 0.35 0.34 7188\n", - "weighted avg 0.81 0.87 0.83 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.89 0.08 0.15 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.89 1.00 0.94 1577\n", - "\n", - " accuracy 0.89 1797\n", - " macro avg 0.59 0.36 0.36 1797\n", - "weighted avg 0.88 0.89 0.84 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 2 and macro avg/f1-score: 0.36267332514096967 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.59 0.42 0.49 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.92 0.96 0.94 6331\n", - "\n", - " accuracy 0.90 7188\n", - " macro avg 0.50 0.46 0.48 7188\n", - "weighted avg 0.88 0.90 0.88 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.61 0.68 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.94 0.98 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.57 0.53 0.55 1797\n", - "weighted avg 0.92 0.93 0.92 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 3 and macro avg/f1-score: 0.5474452879879955 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.61 0.66 0.63 793\n", - " 1 0.25 0.02 0.03 64\n", - " 2 0.95 0.95 0.95 6331\n", - "\n", - " accuracy 0.91 7188\n", - " macro avg 0.60 0.54 0.54 7188\n", - "weighted avg 0.91 0.91 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.68 0.65 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.96 0.95 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.52 0.54 0.53 1797\n", - "weighted avg 0.91 0.91 0.91 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.70 0.66 793\n", - " 1 0.68 0.23 0.35 64\n", - " 2 0.96 0.95 0.96 6331\n", - "\n", - " accuracy 0.92 7188\n", - " macro avg 0.75 0.63 0.65 7188\n", - "weighted avg 0.92 0.92 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.71 0.67 209\n", - " 1 0.45 0.45 0.45 11\n", - " 2 0.96 0.94 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.68 0.70 0.69 1797\n", - "weighted avg 0.92 0.91 0.91 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 5 and macro avg/f1-score: 0.6901427504420462 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.71 0.69 793\n", - " 1 0.60 0.56 0.58 64\n", - " 2 0.96 0.95 0.96 6331\n", - "\n", - " accuracy 0.92 7188\n", - " macro avg 0.74 0.74 0.74 7188\n", - "weighted avg 0.93 0.92 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.71 0.68 209\n", - " 1 0.39 0.64 0.48 11\n", - " 2 0.96 0.94 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.66 0.76 0.70 1797\n", - "weighted avg 0.92 0.91 0.92 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 6 and macro avg/f1-score: 0.7039037826541276 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.72 0.70 793\n", - " 1 0.52 0.53 0.53 64\n", - " 2 0.96 0.96 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.72 0.74 0.73 7188\n", - "weighted avg 0.93 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.63 0.73 0.68 209\n", - " 1 0.41 0.64 0.50 11\n", - " 2 0.96 0.94 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.67 0.77 0.71 1797\n", - "weighted avg 0.92 0.91 0.92 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 7 and macro avg/f1-score: 0.7085684968468765 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.75 0.71 793\n", - " 1 0.62 0.53 0.57 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.74 0.75 7188\n", - "weighted avg 0.93 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.72 0.69 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.96 0.95 0.95 1577\n", - "\n", - " accuracy 0.92 1797\n", - " macro avg 0.68 0.80 0.72 1797\n", - "weighted avg 0.92 0.92 0.92 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 8 and macro avg/f1-score: 0.7214487000648947 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.77 0.73 793\n", - " 1 0.58 0.66 0.62 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.79 0.77 7188\n", - "weighted avg 0.94 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.74 0.71 209\n", - " 1 0.44 0.73 0.55 11\n", - " 2 0.96 0.95 0.96 1577\n", - "\n", - " accuracy 0.92 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.93 0.92 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 9 and macro avg/f1-score: 0.7411771369414074 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.79 0.74 793\n", - " 1 0.65 0.72 0.68 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.77 0.82 0.79 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.74 0.73 209\n", - " 1 0.42 0.73 0.53 11\n", - " 2 0.96 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.93 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.79 0.74 793\n", - " 1 0.63 0.72 0.67 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.77 0.82 0.79 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.75 0.74 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.93 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.81 0.74 793\n", - " 1 0.60 0.66 0.63 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.81 0.78 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.75 0.74 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.94 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.80 0.75 793\n", - " 1 0.60 0.78 0.68 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.76 0.84 0.79 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.77 0.75 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.82 0.74 1797\n", - "weighted avg 0.94 0.93 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 13 and macro avg/f1-score: 0.7440346627353422 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.80 0.75 793\n", - " 1 0.56 0.67 0.61 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.81 0.77 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.76 0.75 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.82 0.74 1797\n", - "weighted avg 0.94 0.93 0.94 1797\n", - "\n" - ] - } - ], "source": [ - "for meta_model in meta_model_names:\n", - " # load and sort out the config\n", - " config_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config.json\")\n", - " with open(config_file, 'r') as jfile:\n", - " config_dict = json.load(jfile)\n", - " config = ConfigMetaCAT()\n", - " for key, value in config_dict.items():\n", - " setattr(config, key, value['py/state']['__dict__'])\n", "\n", - " # change model name if training BERT for the first time\n", - " config.model['model_name'] = 'bert'\n", - "\n", - " tokenizer = TokenizerWrapperBERT.load(os.path.join(base_dir_meta_models,\"meta_\"+meta_model), config.model['model_variant'])\n", - " \n", - " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", - " #Ideally this should replace the meta_models inside the modelpack\n", - "\n", - " #Below are the config values used for Experiencer classification task\n", - "\n", - " config.model['nclasses'] = 3\n", - " config.general['category_name'] = 'Experiencer'\n", - "\n", - " config.train.lr = 5e-4\n", - " config.train['test_size'] = 0.2\n", - " config.train['nepochs'] = 15\n", - "\n", - " # you can also switch between freezing BERT layers or using LoRA during training\n", - " # to use LORA:\n", - " config.model['model_freeze_layers'] = False\n", - "\n", - " config.train.metric['base'] = 'macro avg'\n", - "\n", - " config.train['class_weights'] = [0.4,1.5,0.1]\n", - " config.general['category_value2id'] = {'Family':1, 'Other':0, 'Patient':2}\n", - "\n", - " save_dir_path= \"test_meta\" # Where to save the meta_model and results.\n", - " #Ideally this should replace the meta_models inside the modelpack\n", - "\n", - " # Initialise and train meta_model\n", - " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", - "\n", - " # Save results\n", - " json.dump(results['report'], open(os.path.join(save_dir_path,'meta_results.json'), 'w'))" - ] - }, - { - "cell_type": "markdown", - "id": "8dc9496b", - "metadata": {}, - "source": [ - "BERT model shows improvement for the Family class compared to LSTM, however the recall values can still be improved.
To help tackle this, we'll use 2 phase learning for training." + "#option 1
\n", + "mc.config.train['class_weights'] = []
\n", + "mc.config.train['compute_class_weights'] = True
\n", + "#NOTE: this will only be applicable if mc.config.train.class_weights is empty
\n", + "
\n", + "#2nd option
\n", + "#using specified class weights
\n", + "mc.config.train['class_weights'] = [0.4,0.3,0.1]
" ] }, { "cell_type": "markdown", - "id": "ab23e424", - "metadata": {}, - "source": [ - "## If you dont have the model packs, and are training from scratch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16231060", + "id": "c217762f", "metadata": {}, - "outputs": [], "source": [ - "config = ConfigMetaCAT()\n", - "# make sure to change the following parameters:\n", - "# config.model['nclasses']\n", - "# config.general['category_name']\n", - "\n", - "# change model name if training BERT for the first time\n", - "config.model['model_name'] = 'bert'\n", - "\n", - "tokenizer = TokenizerWrapperBERT.load(\"\", config.model['model_variant'])\n", - "\n", - "save_dir_path= \"test_meta\" # Where to save the meta_model and results. \n", - "#Ideally this should replace the meta_models inside the modelpack\n", - "\n", - "# Initialise and train meta_model\n", - "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", - "\n", - "# Save results\n", - "json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results.json'), 'w'))" + "NOTE: Make sure to correctly map the class weights to their corresponding class index (ID).
To check the index assigned to the classes, use:
`print(mc.config.general['category_value2id'])`\n", + "
This will print a dictionary where the class names and their corresponding IDs (indices) are displayed.
\n", + "The first position in the class weight list corresponds to the class with ID 0 in the dictionary, and so on." ] }, { @@ -1126,1539 +145,81 @@ "id": "c3002ef0", "metadata": {}, "source": [ - "## If using 2 phase learning for training" + "# 2 phase learning for training" ] }, { "cell_type": "markdown", - "id": "b83d9958", + "id": "a349af2b", "metadata": {}, "source": [ - "### Phase 1" + "2 phase learning is used to mitigate class imbalance. In 2 phase learning, the models are trained twice:
\n", + "Phase 1: trains for minority class(es) by undersampling data so that there is no class imbalance\n", + "
Phase 2: trains for all classes\n", + "\n", + "Phase 1 ensures that the model learns minority class(es) and captures the details correctly.\n", + "
Phase 2 is when the model is expected to learn the majority class as it is trained on the entire dataset.\n", + "\n", + "Paper reference - https://ieeexplore.ieee.org/document/7533053\n", + "
NOTE: Make sure to use class weights in favour of minority classes with 2 phase learning" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "9bcdf7be", + "execution_count": null, + "id": "8ff613ef", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", - "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", - "WARNING:medcat.utils.meta_cat.models:\n", - "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", - "INFO:medcat.meta_cat:BERT model used for classification\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {0: 1002, 1: 75, 2: 1002}\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 665\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.48 0.80 0.60 809\n", - " 1 0.03 0.03 0.03 61\n", - " 2 0.44 0.13 0.20 793\n", - "\n", - " accuracy 0.45 1663\n", - " macro avg 0.32 0.32 0.27 1663\n", - "weighted avg 0.44 0.45 0.39 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.47 1.00 0.63 193\n", - " 1 0.00 0.00 0.00 14\n", - " 2 1.00 0.00 0.01 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.49 0.33 0.21 416\n", - "weighted avg 0.72 0.47 0.30 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 0 and macro avg/f1-score: 0.21479741019214701 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.48 0.85 0.62 809\n", - " 1 0.06 0.08 0.07 61\n", - " 2 0.44 0.09 0.14 793\n", - "\n", - " accuracy 0.46 1663\n", - " macro avg 0.33 0.34 0.28 1663\n", - "weighted avg 0.45 0.46 0.37 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.47 1.00 0.63 193\n", - " 1 1.00 0.07 0.13 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.49 0.36 0.26 416\n", - "weighted avg 0.25 0.47 0.30 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 1 and macro avg/f1-score: 0.2560672514619883 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.50 0.89 0.64 809\n", - " 1 0.14 0.36 0.20 61\n", - " 2 0.58 0.05 0.09 793\n", - "\n", - " accuracy 0.47 1663\n", - " macro avg 0.41 0.43 0.31 1663\n", - "weighted avg 0.53 0.47 0.36 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.47 0.97 0.63 193\n", - " 1 0.31 0.36 0.33 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.46 416\n", - " macro avg 0.26 0.44 0.32 416\n", - "weighted avg 0.23 0.46 0.31 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 2 and macro avg/f1-score: 0.322465804759228 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.53 0.90 0.67 809\n", - " 1 0.15 0.66 0.24 61\n", - " 2 0.41 0.01 0.02 793\n", - "\n", - " accuracy 0.47 1663\n", - " macro avg 0.36 0.52 0.31 1663\n", - "weighted avg 0.46 0.47 0.34 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.90 0.71 193\n", - " 1 0.09 0.79 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.44 416\n", - " macro avg 0.23 0.56 0.29 416\n", - "weighted avg 0.27 0.44 0.33 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.56 0.92 0.70 809\n", - " 1 0.15 0.80 0.26 61\n", - " 2 0.75 0.01 0.01 793\n", - "\n", - " accuracy 0.48 1663\n", - " macro avg 0.49 0.58 0.32 1663\n", - "weighted avg 0.64 0.48 0.36 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.60 0.96 0.74 193\n", - " 1 0.08 0.64 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.23 0.53 0.30 416\n", - "weighted avg 0.28 0.47 0.35 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.96 0.73 809\n", - " 1 0.15 0.82 0.26 61\n", - " 2 1.00 0.00 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.58 0.59 0.33 1663\n", - "weighted avg 0.77 0.50 0.36 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.94 0.77 193\n", - " 1 0.07 0.71 0.13 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.46 416\n", - " macro avg 0.24 0.55 0.30 416\n", - "weighted avg 0.31 0.46 0.36 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.63 0.94 0.75 809\n", - " 1 0.13 0.90 0.22 61\n", - " 2 1.00 0.01 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.58 0.62 0.33 1663\n", - "weighted avg 0.79 0.50 0.38 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.95 0.77 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.30 0.47 0.36 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.64 0.95 0.76 809\n", - " 1 0.12 0.93 0.22 61\n", - " 2 0.75 0.00 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.50 0.63 0.33 1663\n", - "weighted avg 0.67 0.50 0.38 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.94 0.77 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.46 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.31 0.46 0.36 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.95 0.78 809\n", - " 1 0.12 0.95 0.21 61\n", - " 2 1.00 0.01 0.02 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.59 0.64 0.34 1663\n", - "weighted avg 0.81 0.50 0.40 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.95 0.78 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.31 0.47 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.95 0.78 809\n", - " 1 0.12 0.95 0.21 61\n", - " 2 1.00 0.00 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.59 0.64 0.33 1663\n", - "weighted avg 0.80 0.50 0.39 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.94 0.78 193\n", - " 1 0.08 0.86 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.31 0.47 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.95 0.79 809\n", - " 1 0.12 0.97 0.21 61\n", - " 2 1.00 0.01 0.02 793\n", - "\n", - " accuracy 0.51 1663\n", - " macro avg 0.60 0.64 0.34 1663\n", - "weighted avg 0.81 0.51 0.40 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.97 0.80 193\n", - " 1 0.09 0.93 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.63 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 10 and macro avg/f1-score: 0.32250773257197235 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.95 0.80 809\n", - " 1 0.11 0.97 0.20 61\n", - " 2 0.77 0.01 0.02 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.52 0.64 0.34 1663\n", - "weighted avg 0.71 0.50 0.41 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.97 0.79 193\n", - " 1 0.09 0.93 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.63 0.32 416\n", - "weighted avg 0.31 0.48 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.96 0.80 809\n", - " 1 0.12 1.00 0.22 61\n", - " 2 1.00 0.02 0.03 793\n", - "\n", - " accuracy 0.51 1663\n", - " macro avg 0.60 0.66 0.35 1663\n", - "weighted avg 0.81 0.51 0.41 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.97 0.80 193\n", - " 1 0.09 0.93 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.63 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 12 and macro avg/f1-score: 0.3237202463008914 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.96 0.80 809\n", - " 1 0.12 0.98 0.22 61\n", - " 2 0.96 0.03 0.06 793\n", - "\n", - " accuracy 0.52 1663\n", - " macro avg 0.59 0.66 0.36 1663\n", - "weighted avg 0.79 0.52 0.42 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.98 0.79 193\n", - " 1 0.09 0.79 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.25 0.59 0.31 416\n", - "weighted avg 0.31 0.48 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.97 0.81 809\n", - " 1 0.12 1.00 0.21 61\n", - " 2 0.88 0.03 0.06 793\n", - "\n", - " accuracy 0.52 1663\n", - " macro avg 0.57 0.67 0.36 1663\n", - "weighted avg 0.77 0.52 0.43 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 193\n", - " 1 0.08 0.86 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.61 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 15 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.97 0.82 809\n", - " 1 0.12 1.00 0.21 61\n", - " 2 0.95 0.05 0.10 793\n", - "\n", - " accuracy 0.53 1663\n", - " macro avg 0.59 0.67 0.38 1663\n", - "weighted avg 0.80 0.53 0.45 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 15 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.81 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.61 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 16 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.97 0.81 809\n", - " 1 0.12 1.00 0.22 61\n", - " 2 0.94 0.04 0.07 793\n", - "\n", - " accuracy 0.53 1663\n", - " macro avg 0.58 0.67 0.37 1663\n", - "weighted avg 0.79 0.53 0.44 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 16 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.80 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 1.00 0.00 0.01 209\n", - "\n", - " accuracy 0.49 416\n", - " macro avg 0.59 0.61 0.32 416\n", - "weighted avg 0.82 0.49 0.38 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 17 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.97 0.81 809\n", - " 1 0.12 1.00 0.22 61\n", - " 2 0.94 0.06 0.12 793\n", - "\n", - " accuracy 0.54 1663\n", - " macro avg 0.59 0.68 0.39 1663\n", - "weighted avg 0.80 0.54 0.46 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 17 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.98 0.79 193\n", - " 1 0.09 0.86 0.17 14\n", - " 2 1.00 0.02 0.04 209\n", - "\n", - " accuracy 0.49 416\n", - " macro avg 0.59 0.62 0.33 416\n", - "weighted avg 0.81 0.49 0.39 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 17 and macro avg/f1-score: 0.3328554768276514 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 18 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 809\n", - " 1 0.14 1.00 0.24 61\n", - " 2 0.95 0.08 0.15 793\n", - "\n", - " accuracy 0.55 1663\n", - " macro avg 0.59 0.69 0.40 1663\n", - "weighted avg 0.80 0.55 0.47 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 18 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.98 0.80 193\n", - " 1 0.09 0.86 0.17 14\n", - " 2 1.00 0.04 0.07 209\n", - "\n", - " accuracy 0.50 416\n", - " macro avg 0.59 0.62 0.35 416\n", - "weighted avg 0.82 0.50 0.41 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 18 and macro avg/f1-score: 0.3471379464272786 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 19 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.98 0.82 809\n", - " 1 0.13 1.00 0.24 61\n", - " 2 0.97 0.09 0.17 793\n", - "\n", - " accuracy 0.56 1663\n", - " macro avg 0.60 0.69 0.41 1663\n", - "weighted avg 0.81 0.56 0.49 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 19 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.80 193\n", - " 1 0.10 0.86 0.17 14\n", - " 2 1.00 0.07 0.13 209\n", - "\n", - " accuracy 0.52 416\n", - " macro avg 0.59 0.63 0.37 416\n", - "weighted avg 0.82 0.52 0.44 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 19 and macro avg/f1-score: 0.3674925759230418 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 20 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.98 0.83 809\n", - " 1 0.13 1.00 0.23 61\n", - " 2 0.94 0.10 0.18 793\n", - "\n", - " accuracy 0.56 1663\n", - " macro avg 0.59 0.69 0.41 1663\n", - "weighted avg 0.80 0.56 0.49 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 20 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 193\n", - " 1 0.10 0.86 0.18 14\n", - " 2 1.00 0.10 0.17 209\n", - "\n", - " accuracy 0.53 416\n", - " macro avg 0.60 0.64 0.39 416\n", - "weighted avg 0.83 0.53 0.47 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 20 and macro avg/f1-score: 0.38685497295333987 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 21 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.98 0.81 809\n", - " 1 0.14 1.00 0.25 61\n", - " 2 0.96 0.12 0.22 793\n", - "\n", - " accuracy 0.57 1663\n", - " macro avg 0.60 0.70 0.43 1663\n", - "weighted avg 0.80 0.57 0.51 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 21 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.81 193\n", - " 1 0.11 0.86 0.19 14\n", - " 2 0.96 0.11 0.21 209\n", - "\n", - " accuracy 0.54 416\n", - " macro avg 0.58 0.65 0.40 416\n", - "weighted avg 0.80 0.54 0.48 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 21 and macro avg/f1-score: 0.4002297137890358 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 22 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.99 0.82 809\n", - " 1 0.14 1.00 0.25 61\n", - " 2 0.98 0.12 0.22 793\n", - "\n", - " accuracy 0.57 1663\n", - " macro avg 0.61 0.70 0.43 1663\n", - "weighted avg 0.81 0.57 0.51 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 22 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 193\n", - " 1 0.11 0.86 0.20 14\n", - " 2 0.94 0.15 0.26 209\n", - "\n", - " accuracy 0.56 416\n", - " macro avg 0.58 0.66 0.42 416\n", - "weighted avg 0.80 0.56 0.51 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 22 and macro avg/f1-score: 0.4231725462046246 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 23 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.99 0.83 809\n", - " 1 0.15 1.00 0.26 61\n", - " 2 0.99 0.17 0.29 793\n", - "\n", - " accuracy 0.60 1663\n", - " macro avg 0.62 0.72 0.46 1663\n", - "weighted avg 0.82 0.60 0.55 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 23 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.99 0.81 193\n", - " 1 0.12 0.86 0.21 14\n", - " 2 1.00 0.19 0.31 209\n", - "\n", - " accuracy 0.58 416\n", - " macro avg 0.60 0.68 0.45 416\n", - "weighted avg 0.83 0.58 0.54 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 23 and macro avg/f1-score: 0.4459035717026967 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 24 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.99 0.83 809\n", - " 1 0.16 1.00 0.28 61\n", - " 2 0.99 0.21 0.35 793\n", - "\n", - " accuracy 0.62 1663\n", - " macro avg 0.62 0.73 0.49 1663\n", - "weighted avg 0.83 0.62 0.58 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 24 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.98 0.83 193\n", - " 1 0.12 0.86 0.21 14\n", - " 2 0.98 0.25 0.40 209\n", - "\n", - " accuracy 0.61 416\n", - " macro avg 0.61 0.70 0.48 416\n", - "weighted avg 0.83 0.61 0.60 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 24 and macro avg/f1-score: 0.482945795402635 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 25 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.99 0.83 809\n", - " 1 0.15 1.00 0.27 61\n", - " 2 0.98 0.18 0.31 793\n", - "\n", - " accuracy 0.60 1663\n", - " macro avg 0.62 0.72 0.47 1663\n", - "weighted avg 0.82 0.60 0.56 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 25 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.98 0.83 193\n", - " 1 0.13 0.86 0.22 14\n", - " 2 0.98 0.27 0.43 209\n", - "\n", - " accuracy 0.62 416\n", - " macro avg 0.61 0.70 0.49 416\n", - "weighted avg 0.83 0.62 0.61 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 25 and macro avg/f1-score: 0.493566120394729 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 26 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.99 0.83 809\n", - " 1 0.17 1.00 0.29 61\n", - " 2 0.99 0.24 0.39 793\n", - "\n", - " accuracy 0.63 1663\n", - " macro avg 0.63 0.74 0.50 1663\n", - "weighted avg 0.83 0.63 0.60 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 26 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.98 0.84 193\n", - " 1 0.14 0.86 0.24 14\n", - " 2 0.97 0.34 0.50 209\n", - "\n", - " accuracy 0.65 416\n", - " macro avg 0.62 0.73 0.53 416\n", - "weighted avg 0.84 0.65 0.65 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 26 and macro avg/f1-score: 0.52768022857349 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 27 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.98 0.85 809\n", - " 1 0.17 1.00 0.29 61\n", - " 2 0.98 0.28 0.43 793\n", - "\n", - " accuracy 0.65 1663\n", - " macro avg 0.63 0.75 0.52 1663\n", - "weighted avg 0.84 0.65 0.63 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 27 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.98 0.84 193\n", - " 1 0.15 0.86 0.25 14\n", - " 2 0.97 0.37 0.53 209\n", - "\n", - " accuracy 0.67 416\n", - " macro avg 0.62 0.73 0.54 416\n", - "weighted avg 0.84 0.67 0.67 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 27 and macro avg/f1-score: 0.5428240740740741 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 28 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.99 0.85 809\n", - " 1 0.18 1.00 0.31 61\n", - " 2 0.98 0.31 0.48 793\n", - "\n", - " accuracy 0.67 1663\n", - " macro avg 0.64 0.77 0.54 1663\n", - "weighted avg 0.84 0.67 0.65 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 28 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.98 0.86 193\n", - " 1 0.15 0.86 0.26 14\n", - " 2 0.98 0.41 0.58 209\n", - "\n", - " accuracy 0.69 416\n", - " macro avg 0.63 0.75 0.56 416\n", - "weighted avg 0.85 0.69 0.70 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 28 and macro avg/f1-score: 0.5638621950678688 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 29 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.99 0.85 809\n", - " 1 0.19 1.00 0.32 61\n", - " 2 0.99 0.33 0.50 793\n", - "\n", - " accuracy 0.68 1663\n", - " macro avg 0.64 0.77 0.55 1663\n", - "weighted avg 0.84 0.68 0.66 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 29 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.98 0.85 193\n", - " 1 0.16 0.86 0.27 14\n", - " 2 0.98 0.43 0.59 209\n", - "\n", - " accuracy 0.70 416\n", - " macro avg 0.63 0.75 0.57 416\n", - "weighted avg 0.85 0.70 0.70 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 29 and macro avg/f1-score: 0.5720897974597215 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 30 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.99 0.86 809\n", - " 1 0.18 1.00 0.30 61\n", - " 2 0.99 0.34 0.51 793\n", - "\n", - " accuracy 0.68 1663\n", - " macro avg 0.64 0.78 0.56 1663\n", - "weighted avg 0.85 0.68 0.67 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 30 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.98 0.86 193\n", - " 1 0.17 0.86 0.29 14\n", - " 2 0.98 0.48 0.65 209\n", - "\n", - " accuracy 0.73 416\n", - " macro avg 0.64 0.77 0.60 416\n", - "weighted avg 0.86 0.73 0.74 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 30 and macro avg/f1-score: 0.6005270274314415 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 31 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.99 0.87 809\n", - " 1 0.21 1.00 0.35 61\n", - " 2 0.99 0.43 0.60 793\n", - "\n", - " accuracy 0.72 1663\n", - " macro avg 0.66 0.81 0.61 1663\n", - "weighted avg 0.86 0.72 0.72 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 31 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.98 0.86 193\n", - " 1 0.21 0.86 0.33 14\n", - " 2 0.98 0.54 0.69 209\n", - "\n", - " accuracy 0.75 416\n", - " macro avg 0.65 0.79 0.63 416\n", - "weighted avg 0.86 0.75 0.76 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 31 and macro avg/f1-score: 0.6306067812327066 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 32 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.98 0.87 809\n", - " 1 0.23 1.00 0.37 61\n", - " 2 0.98 0.46 0.63 793\n", - "\n", - " accuracy 0.73 1663\n", - " macro avg 0.66 0.81 0.62 1663\n", - "weighted avg 0.85 0.73 0.73 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 32 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.98 0.87 193\n", - " 1 0.21 0.86 0.34 14\n", - " 2 0.98 0.56 0.71 209\n", - "\n", - " accuracy 0.76 416\n", - " macro avg 0.66 0.80 0.64 416\n", - "weighted avg 0.87 0.76 0.77 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 32 and macro avg/f1-score: 0.6414740059803422 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 33 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.99 0.87 809\n", - " 1 0.23 1.00 0.38 61\n", - " 2 0.99 0.46 0.63 793\n", - "\n", - " accuracy 0.74 1663\n", - " macro avg 0.67 0.82 0.62 1663\n", - "weighted avg 0.86 0.74 0.74 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 33 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.98 0.87 193\n", - " 1 0.22 0.86 0.35 14\n", - " 2 0.98 0.57 0.72 209\n", - "\n", - " accuracy 0.77 416\n", - " macro avg 0.66 0.80 0.65 416\n", - "weighted avg 0.87 0.77 0.78 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 33 and macro avg/f1-score: 0.6473391409830812 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 34 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.98 0.86 809\n", - " 1 0.23 1.00 0.38 61\n", - " 2 0.99 0.45 0.62 793\n", - "\n", - " accuracy 0.73 1663\n", - " macro avg 0.66 0.81 0.62 1663\n", - "weighted avg 0.85 0.73 0.73 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 34 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.98 0.88 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.99 0.63 0.77 209\n", - "\n", - " accuracy 0.80 416\n", - " macro avg 0.68 0.82 0.69 416\n", - "weighted avg 0.87 0.80 0.81 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 34 and macro avg/f1-score: 0.6868341887176047 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 35 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.78 0.99 0.87 809\n", - " 1 0.26 1.00 0.41 61\n", - " 2 0.99 0.51 0.67 793\n", - "\n", - " accuracy 0.76 1663\n", - " macro avg 0.68 0.83 0.65 1663\n", - "weighted avg 0.86 0.76 0.76 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 35 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.23 0.86 0.36 14\n", - " 2 0.98 0.62 0.76 209\n", - "\n", - " accuracy 0.79 416\n", - " macro avg 0.67 0.82 0.67 416\n", - "weighted avg 0.88 0.79 0.80 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 36 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.99 0.88 809\n", - " 1 0.24 1.00 0.39 61\n", - " 2 0.99 0.50 0.66 793\n", - "\n", - " accuracy 0.75 1663\n", - " macro avg 0.67 0.83 0.64 1663\n", - "weighted avg 0.87 0.75 0.76 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 36 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.69 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 36 and macro avg/f1-score: 0.6928908001266963 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 37 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.98 0.88 809\n", - " 1 0.26 1.00 0.41 61\n", - " 2 0.98 0.53 0.69 793\n", - "\n", - " accuracy 0.77 1663\n", - " macro avg 0.68 0.84 0.66 1663\n", - "weighted avg 0.87 0.77 0.77 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 37 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.70 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 37 and macro avg/f1-score: 0.6963856616166311 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 38 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.78 0.98 0.87 809\n", - " 1 0.25 1.00 0.41 61\n", - " 2 0.98 0.50 0.66 793\n", - "\n", - " accuracy 0.75 1663\n", - " macro avg 0.67 0.83 0.65 1663\n", - "weighted avg 0.86 0.75 0.75 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 38 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.70 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 39 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.98 0.88 809\n", - " 1 0.26 1.00 0.41 61\n", - " 2 0.98 0.53 0.69 793\n", - "\n", - " accuracy 0.77 1663\n", - " macro avg 0.68 0.84 0.66 1663\n", - "weighted avg 0.86 0.77 0.77 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 39 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.70 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "######################################################################################################\n", - "# 2 phase learning (used for imbalanced datasets) - trains the models twice: \n", - "# phase 1: trains for minority class(es) by undersampling data\n", - "# phase 2: trains for all classes\n", - "# parameter values: \n", - "# 1: Phase 1 - Train model on undersampled data\n", - "# 2: Phase 2 - Continue training on full data\n", - "# 0: None\n", - "#\n", - "# Paper reference - https://ieeexplore.ieee.org/document/7533053\n", - "# NOTE: Make sure to use class weights in favour of minority classes with 2 phase learning\n", - "#####################################################################################################\n", - "\n", - "# Follow same steps till defining save_dir_path\n", - "\n", - "#change phase number to 1\n", - "config.model.phase_number = 1\n", + "#--------------------------------Phase 1--------------------------------\n", + "def run_phase_1(meta_model,class_wt_phase1 = None):\n", + " #Loading the pre-defined config for phase 1\n", + " config_ph_1_path = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config_ph1.json\")\n", + " with open(config_ph_1_path) as f:\n", + " config_ph1 = json.load(f)\n", "\n", - "# specify the class that will define the desired sample size for the undersampling process\n", - "# if this is left empty, the class with the lowest samples will be chosen\n", - "# example\n", - "config.model['category_undersample'] = 'Other'\n", + " mc = MetaCAT.load(save_dir_path=os.path.join(base_dir_meta_models,\"meta_\"+meta_model),config_dict = config_ph1)\n", "\n", - "#Below are the config values used for Experiencer classification task\n", + " if class_wt_phase1:\n", + " mc.config.train['class_weights'] = class_wt_phase1\n", "\n", - "config.model['nclasses'] = 3\n", - "config.general['category_name'] = 'Experiencer'\n", + " mc.config.train['nepochs'] = 30 #You can change the number of epochs, remember to keep them higher for phase 1\n", "\n", - "config.model['category_undersample'] = 'Other'\n", - "\n", - "config.train.lr = 5e-4\n", - "config.train['test_size'] = 0.2\n", - "config.train['nepochs'] = 20\n", - "\n", - "config.train.metric['base'] = 'macro avg'\n", - "\n", - "config.train['class_weights'] = [0.4,1.5,0.05]\n", - "config.general['category_value2id'] = {'Other':0, 'Family':1, 'Patient':2}\n", + " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", + " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", + " # Save results\n", + " json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results_phase1.json'), 'w'))\n", "\n", - "config.model['model_freeze_layers'] = False\n", + "#--------------------------------Phase 2--------------------------------\n", + "def run_phase_2(meta_model,class_wt_phase2 = None): \n", + " #Loading the pre-defined config for phase 2\n", + " config_ph_2_path = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config_ph2.json\")\n", + " with open(config_ph_2_path) as f:\n", + " config_ph2 = json.load(f)\n", "\n", - "# Initialise and train meta_model \n", - "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", + " mc = MetaCAT.load(save_dir_path=os.path.join(base_dir_meta_models,\"meta_\"+meta_model),config_dict = config_ph2)\n", "\n", - "# Save results\n", - "json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results_phase1.json'), 'w'))" - ] - }, - { - "cell_type": "markdown", - "id": "e229ef0a", - "metadata": {}, - "source": [ - "## Phase 2" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "63c01002", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", - "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", - "WARNING:medcat.utils.meta_cat.models:\n", - "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", - "INFO:medcat.meta_cat:BERT model used for classification\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {0: 1002, 1: 75, 2: 1002}\n", - "INFO:medcat.meta_cat:Model state loaded from dict for 2 phase learning\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 718\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.32 0.94 0.48 793\n", - " 1 0.07 0.97 0.13 64\n", - " 2 0.99 0.62 0.76 6331\n", - "\n", - " accuracy 0.66 7188\n", - " macro avg 0.46 0.84 0.45 7188\n", - "weighted avg 0.91 0.66 0.72 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.40 0.95 0.56 209\n", - " 1 0.10 1.00 0.18 11\n", - " 2 0.99 0.75 0.85 1577\n", - "\n", - " accuracy 0.77 1797\n", - " macro avg 0.50 0.90 0.53 1797\n", - "weighted avg 0.92 0.77 0.81 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.42 0.92 0.58 793\n", - " 1 0.15 0.97 0.26 64\n", - " 2 0.99 0.79 0.88 6331\n", - "\n", - " accuracy 0.80 7188\n", - " macro avg 0.52 0.89 0.57 7188\n", - "weighted avg 0.92 0.80 0.84 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.49 0.92 0.64 209\n", - " 1 0.12 1.00 0.21 11\n", - " 2 0.99 0.82 0.90 1577\n", - "\n", - " accuracy 0.83 1797\n", - " macro avg 0.53 0.91 0.58 1797\n", - "weighted avg 0.92 0.83 0.86 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.48 0.92 0.63 793\n", - " 1 0.21 0.97 0.35 64\n", - " 2 0.99 0.84 0.91 6331\n", - "\n", - " accuracy 0.85 7188\n", - " macro avg 0.56 0.91 0.63 7188\n", - "weighted avg 0.93 0.85 0.87 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.49 0.94 0.64 209\n", - " 1 0.13 1.00 0.23 11\n", - " 2 0.99 0.82 0.90 1577\n", - "\n", - " accuracy 0.84 1797\n", - " macro avg 0.54 0.92 0.59 1797\n", - "weighted avg 0.93 0.84 0.87 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.53 0.92 0.67 793\n", - " 1 0.25 0.95 0.40 64\n", - " 2 0.99 0.87 0.92 6331\n", - "\n", - " accuracy 0.87 7188\n", - " macro avg 0.59 0.91 0.66 7188\n", - "weighted avg 0.93 0.87 0.89 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.55 0.92 0.69 209\n", - " 1 0.14 1.00 0.25 11\n", - " 2 0.99 0.86 0.92 1577\n", - "\n", - " accuracy 0.87 1797\n", - " macro avg 0.56 0.93 0.62 1797\n", - "weighted avg 0.93 0.87 0.89 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.53 0.93 0.68 793\n", - " 1 0.31 0.95 0.47 64\n", - " 2 0.99 0.88 0.93 6331\n", - "\n", - " accuracy 0.88 7188\n", - " macro avg 0.61 0.92 0.69 7188\n", - "weighted avg 0.93 0.88 0.90 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.56 0.92 0.70 209\n", - " 1 0.15 1.00 0.27 11\n", - " 2 0.99 0.87 0.92 1577\n", - "\n", - " accuracy 0.87 1797\n", - " macro avg 0.57 0.93 0.63 1797\n", - "weighted avg 0.93 0.87 0.89 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.56 0.92 0.70 793\n", - " 1 0.32 0.92 0.47 64\n", - " 2 0.99 0.89 0.94 6331\n", - "\n", - " accuracy 0.89 7188\n", - " macro avg 0.62 0.91 0.70 7188\n", - "weighted avg 0.94 0.89 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.57 0.92 0.71 209\n", - " 1 0.17 1.00 0.29 11\n", - " 2 0.99 0.87 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.58 0.93 0.64 1797\n", - "weighted avg 0.93 0.88 0.90 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.92 0.71 793\n", - " 1 0.34 0.95 0.50 64\n", - " 2 0.99 0.90 0.94 6331\n", - "\n", - " accuracy 0.90 7188\n", - " macro avg 0.63 0.92 0.72 7188\n", - "weighted avg 0.94 0.90 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.90 0.71 209\n", - " 1 0.16 1.00 0.28 11\n", - " 2 0.99 0.88 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.58 0.93 0.64 1797\n", - "weighted avg 0.93 0.88 0.90 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.59 0.93 0.72 793\n", - " 1 0.32 0.97 0.48 64\n", - " 2 0.99 0.90 0.94 6331\n", - "\n", - " accuracy 0.90 7188\n", - " macro avg 0.63 0.93 0.71 7188\n", - "weighted avg 0.94 0.90 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.60 0.89 0.72 209\n", - " 1 0.19 1.00 0.31 11\n", - " 2 0.98 0.89 0.94 1577\n", - "\n", - " accuracy 0.89 1797\n", - " macro avg 0.59 0.93 0.66 1797\n", - "weighted avg 0.94 0.89 0.91 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.59 0.92 0.72 793\n", - " 1 0.35 0.97 0.51 64\n", - " 2 0.99 0.90 0.94 6331\n", - "\n", - " accuracy 0.91 7188\n", - " macro avg 0.64 0.93 0.73 7188\n", - "weighted avg 0.94 0.91 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.89 0.73 209\n", - " 1 0.20 1.00 0.33 11\n", - " 2 0.98 0.90 0.94 1577\n", - "\n", - " accuracy 0.90 1797\n", - " macro avg 0.60 0.93 0.67 1797\n", - "weighted avg 0.94 0.90 0.91 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.92 0.74 793\n", - " 1 0.36 0.95 0.52 64\n", - " 2 0.99 0.91 0.95 6331\n", - "\n", - " accuracy 0.91 7188\n", - " macro avg 0.66 0.93 0.74 7188\n", - "weighted avg 0.94 0.91 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.64 0.88 0.74 209\n", - " 1 0.21 1.00 0.34 11\n", - " 2 0.98 0.91 0.94 1577\n", - "\n", - " accuracy 0.90 1797\n", - " macro avg 0.61 0.93 0.68 1797\n", - "weighted avg 0.94 0.90 0.92 1797\n", - "\n" - ] - } - ], - "source": [ - "# Perform 2nd round of training\n", + " if class_wt_phase2:\n", + " mc.config.train['class_weights'] = class_wt_phase2\n", "\n", - "config.model['phase_number'] = 2\n", - "config.train['class_weights'] = [0.3,1,0.05]\n", - "config.train['nepochs'] = 10\n", + " mc.config.train['nepochs'] = 15\n", "\n", - "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", + " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. Ensure to keep this same as Phase 1\n", + " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", + " # Save results\n", + " json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results_phase2.json'), 'w'))\n", "\n", - "# Save results\n", - "json.dump(results['report'], open(os.path.join(save_dir_path,'meta_'+meta_model+'_results_phase2.json'), 'w'))" - ] - }, - { - "cell_type": "markdown", - "id": "62771904", - "metadata": {}, - "source": [ - "Using 2 phase learning boosts the performance of the model, especially for the minority classes, wiht 0.88 and 1 recall values.
This highlights the impact of using 2 phase learning for model training with imbalanced datasets\n", - "

NOTE: The observed performance improvements are dataset-dependent, and you may not experience such substantial gains. Additionally, class weights and other hyperparameters will need to be fine-tuned for your specific dataset." + "#--------------------------------Driver--------------------------------\n", + "for meta_model in meta_model_names:\n", + " #To use your own class weights instead of the pre-defined ones for the 2 phases, uncomment the below lines\n", + " '''class_wt_phase1 = []\n", + " class_wt_phase2 = []'''\n", + "\n", + " # Train 2 phase learning\n", + " logger.info(\"\\n********************Beginning Phase 1********************\")\n", + " run_phase_1(meta_model,class_wt_phase1)\n", + " logger.info(\"\\n********************Beginning Phase 2********************\")\n", + " run_phase_2(meta_model,class_wt_phase2)" ] }, {