multimodal-transformer-embedding-keras.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Please note that despite the accuracy of this notebook, it used a BERT transformer for text encoding. \n## So, it is ineligible on a technicality for using an online resource.\n\n## However, I was able to achieve validation accuracy scores between 85% to 95% with this method and I thought I would publish it.","metadata":{}},{"cell_type":"markdown","source":"# It was rough to not be able to use this in the competition so please leave an upvote if you found this interesting! 🥲","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ndf_test=pd.read_csv('../input/us-patent-phrase-to-phrase-matching/test.csv')\nprint(df_train.shape)\nprint(df_test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_col=df_train.columns\ndf_test_col=df_test.columns\n\ndf_train['set']='train'\ndf_test['set']='test'\ndf_test['score']=0\n\ndf_train_test=df_train.append(df_test).reset_index(drop=True)\ntraindummy = pd.get_dummies(df_train_test['context'], prefix='context_')\ndf_train_test = pd.merge(\n    df_train_test,\n    traindummy,\n    left_index=True,\n    right_index=True,)\n\ncat_var_list=[]\nfor col in df_train_test.columns:\n  if 'context__' in col:\n    cat_var_list.append(col)\nprint(len(cat_var_list))\n\ndf_train=df_train_test[df_train_test['set']=='train'].reset_index(drop=True)\ndf_test=df_train_test[df_train_test['set']=='test'].reset_index(drop=True)\ndf_train=df_train[[*df_train_col, *cat_var_list]]\ndf_test=df_test[[*df_test_col, *cat_var_list]]\nprint(df_train.shape)\nprint(df_test.shape)","metadata":{"id":"k8ilT8tRZGPr","outputId":"7e1191ae-90a0-471e-b42f-af57ac329b8e","execution":{"iopub.status.busy":"2022-04-28T04:54:57.263475Z","iopub.execute_input":"2022-04-28T04:54:57.263793Z","iopub.status.idle":"2022-04-28T04:54:57.416759Z","shell.execute_reply.started":"2022-04-28T04:54:57.263763Z","shell.execute_reply":"2022-04-28T04:54:57.41563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"id":"lHYh7SjLZKoF","outputId":"2301604b-4e9d-4666-ce54-56e2fea4493a","execution":{"iopub.status.busy":"2022-04-28T04:54:58.222017Z","iopub.execute_input":"2022-04-28T04:54:58.222297Z","iopub.status.idle":"2022-04-28T04:54:58.251317Z","shell.execute_reply.started":"2022-04-28T04:54:58.222268Z","shell.execute_reply":"2022-04-28T04:54:58.250417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"id":"B51Dxr9sQREF","outputId":"b27d9bfe-694b-4b4e-f749-6fb85b9e6e1a","execution":{"iopub.status.busy":"2022-04-28T04:54:58.835825Z","iopub.execute_input":"2022-04-28T04:54:58.836669Z","iopub.status.idle":"2022-04-28T04:54:58.859311Z","shell.execute_reply.started":"2022-04-28T04:54:58.836627Z","shell.execute_reply":"2022-04-28T04:54:58.85844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tensorflow_text\n!pip install tensorflow_hub","metadata":{"id":"OnGW175VAHm8","outputId":"5cde2a82-e586-4c5d-a5ce-70d2480e8ea2","execution":{"iopub.status.busy":"2022-04-28T04:54:59.519415Z","iopub.execute_input":"2022-04-28T04:54:59.519694Z","iopub.status.idle":"2022-04-28T04:56:18.291153Z","shell.execute_reply.started":"2022-04-28T04:54:59.519666Z","shell.execute_reply":"2022-04-28T04:56:18.289128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_text as text\nimport tensorflow_hub as hub\nfrom tensorflow import keras\nfrom sklearn.model_selection import train_test_split\n\ntrain_dfsam, val_df = train_test_split(df_train, test_size=0.10, stratify=df_train[\"context\"].values, random_state=42)","metadata":{"id":"JkU9o39dQF5W","execution":{"iopub.status.busy":"2022-04-28T04:56:29.34354Z","iopub.execute_input":"2022-04-28T04:56:29.343979Z","iopub.status.idle":"2022-04-28T04:56:34.946252Z","shell.execute_reply.started":"2022-04-28T04:56:29.343927Z","shell.execute_reply":"2022-04-28T04:56:34.945181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.tensorflow.org/text/tutorials/bert_glue#loading_models_from_tensorflow_hub\n\n#lite:\nbert_model_path=\"https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1\"\n\n# heavy\n# bert_model_path=\"https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_large/2\"\n\nbert_preprocess_path=\"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\"","metadata":{"id":"Cbzp4BLZBoqQ","execution":{"iopub.status.busy":"2022-04-28T04:56:34.948856Z","iopub.execute_input":"2022-04-28T04:56:34.949106Z","iopub.status.idle":"2022-04-28T04:56:34.954975Z","shell.execute_reply.started":"2022-04-28T04:56:34.949075Z","shell.execute_reply":"2022-04-28T04:56:34.953602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_bert_preprocessing_model(sentence_features, seq_length=128):\n    input_segments = [\n        tf.keras.layers.Input(shape=(), dtype=tf.string, name=ft)\n        for ft in sentence_features]\n    bert_preprocess = hub.load(bert_preprocess_path)\n    tokenizer = hub.KerasLayer(bert_preprocess.tokenize, name=\"tokenizer\")\n    segments = [tokenizer(s) for s in input_segments]\n    truncated_segments = segments\n    packer = hub.KerasLayer(bert_preprocess.bert_pack_inputs, arguments=dict(seq_length=seq_length), name=\"packer\")\n    model_inputs = packer(truncated_segments)\n    return keras.Model(input_segments, model_inputs)\n\n\nbert_preprocess_model = make_bert_preprocessing_model([\"text_1\", \"text_2\"])\nkeras.utils.plot_model(bert_preprocess_model, show_shapes=True, show_dtype=True)","metadata":{"id":"Qi7ii-cdTYJn","outputId":"d16c3439-27ec-45d5-c19c-c8c1b20d3228","execution":{"iopub.status.busy":"2022-04-28T04:56:34.95696Z","iopub.execute_input":"2022-04-28T04:56:34.957573Z","iopub.status.idle":"2022-04-28T04:56:39.803772Z","shell.execute_reply.started":"2022-04-28T04:56:34.957537Z","shell.execute_reply":"2022-04-28T04:56:39.802966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_text(text_1_series, text_2_series):\n    output = bert_preprocess_model([np.array(text_1_series.to_list()), np.array(text_2_series.to_list())])\n    output = {feature: tf.squeeze(output[feature]) for feature in bert_input_features}\n    return output","metadata":{"id":"OPzLU6bFcanq","execution":{"iopub.status.busy":"2022-04-28T04:56:39.805943Z","iopub.execute_input":"2022-04-28T04:56:39.806406Z","iopub.status.idle":"2022-04-28T04:56:39.812421Z","shell.execute_reply.started":"2022-04-28T04:56:39.80637Z","shell.execute_reply":"2022-04-28T04:56:39.811285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def project_embeddings(embeddings, num_projection_layers, projection_dims, dropout_rate):\n    projected_embeddings = keras.layers.Dense(units=projection_dims)(embeddings)\n    for _ in range(num_projection_layers):\n        x = tf.nn.gelu(projected_embeddings)\n        x = keras.layers.Dense(projection_dims)(x)\n        x = keras.layers.Dropout(dropout_rate)(x)\n        x = keras.layers.Add()([projected_embeddings, x])\n        projected_embeddings = keras.layers.LayerNormalization()(x)\n    return projected_embeddings\ndef create_text_encoder(\n    num_projection_layers, projection_dims, dropout_rate, trainable=False\n):\n    bert = hub.KerasLayer(bert_model_path, name=\"bert\",)\n    bert.trainable = trainable\n\n    bert_input_features = [\"input_type_ids\", \"input_mask\", \"input_word_ids\"]\n    inputs = {\n        feature: keras.Input(shape=(128,), dtype=tf.int32, name=feature)\n        for feature in bert_input_features\n    }\n    embeddings = bert(inputs)[\"pooled_output\"]\n    outputs = project_embeddings(\n        embeddings, num_projection_layers, projection_dims, dropout_rate\n    )\n    return keras.Model(inputs, outputs, name=\"text_encoder\")","metadata":{"id":"jAKyds2UfUfa","execution":{"iopub.status.busy":"2022-04-28T04:56:39.814015Z","iopub.execute_input":"2022-04-28T04:56:39.814351Z","iopub.status.idle":"2022-04-28T04:56:39.830092Z","shell.execute_reply.started":"2022-04-28T04:56:39.814309Z","shell.execute_reply":"2022-04-28T04:56:39.829184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\n\nbert_input_features = [\"input_type_ids\", \"input_mask\", \"input_word_ids\"]\ntext_inputs = {\n    feature: keras.Input(shape=(128,), dtype=tf.int32, name=feature)\n    for feature in bert_input_features}\n\ntext_encoder = create_text_encoder(1,256,.1,True)\ntext_projections = text_encoder(text_inputs)\ntext_projections","metadata":{"id":"G2JZr4C5XagN","outputId":"0ed50117-5fc2-4d58-c497-d66170022c03","execution":{"iopub.status.busy":"2022-04-28T04:56:39.831788Z","iopub.execute_input":"2022-04-28T04:56:39.832285Z","iopub.status.idle":"2022-04-28T04:56:45.272198Z","shell.execute_reply.started":"2022-04-28T04:56:39.832243Z","shell.execute_reply":"2022-04-28T04:56:45.271352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import *\n##possibly try leaky relu\ncat_var=keras.Input(shape=(len(cat_var_list),))\ntransformer_embedding = keras.layers.Concatenate()([text_encoder(text_inputs), cat_var])\ndense1 = keras.layers.Dense(256, activation=\"relu\")(transformer_embedding)\ndropout1 = keras.layers.Dropout(0.2)(dense1)\ndense2 = keras.layers.Dense(128, activation=\"relu\")(dropout1)\ndropout2 = keras.layers.Dropout(0.1)(dense2)\ndense3 = keras.layers.Dense(64, activation=\"relu\")(dropout2)\ndropout3 = keras.layers.Dropout(0.05)(dense3)\ndense4 = keras.layers.Dense(16, activation=\"relu\")(dropout3)\noutputs = keras.layers.Dense(1, activation=\"sigmoid\")(dense4)\n\nmodel=keras.Model([text_inputs, cat_var], outputs)\nkeras.utils.plot_model(model, show_shapes=True)\n","metadata":{"id":"l-ybCvNXrD28","outputId":"b4e6d081-827a-4255-91ae-fd0757da57b1","execution":{"iopub.status.busy":"2022-04-28T04:56:45.273798Z","iopub.execute_input":"2022-04-28T04:56:45.2745Z","iopub.status.idle":"2022-04-28T04:56:46.087849Z","shell.execute_reply.started":"2022-04-28T04:56:45.274447Z","shell.execute_reply":"2022-04-28T04:56:46.087087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#will use pearson correlation coefficent for loss function as the competition is using it for an accuracy score:\n# tf.enable_eager_execution()\n\ndef t(a):\n  return tf.constant(a, dtype=tf.float64)\ndef tmean(x, axis=-1):\n  x = tf.convert_to_tensor(x)\n  sum = tf.reduce_sum(x, axis=axis)\n  n = tf.cast(tf.shape(x)[axis], x.dtype)\n  return sum / n\n\ntmean(t([[1.0],[2.0],[3.0]]), axis=-2)","metadata":{"id":"U8CC1OghSpmn","outputId":"edb02e9d-58b5-42f8-ed59-da9322c0e3f0","execution":{"iopub.status.busy":"2022-04-28T04:56:46.090675Z","iopub.execute_input":"2022-04-28T04:56:46.090982Z","iopub.status.idle":"2022-04-28T04:56:46.110283Z","shell.execute_reply.started":"2022-04-28T04:56:46.09094Z","shell.execute_reply":"2022-04-28T04:56:46.109259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pearson correlation coefficlent : $r_{xy} = \\frac{\\sum\\left((x-\\overline{x})(y-\\overline{y})\\right)}{\\sqrt{\\sum(x-\\overline{x})^2\\sum(y-\\overline{y})^2}} $","metadata":{"id":"we7wcOLlTB2M"}},{"cell_type":"code","source":"from tensorflow.python.ops import math_ops\n# from tensorflow.python.ops import squared_difference\n\ndef correlationLoss(x,y, axis=-2):\n  x = tf.convert_to_tensor(x)\n  y = math_ops.cast(y, x.dtype)\n  n = tf.cast(tf.shape(x)[axis], x.dtype)\n  xsum = tf.reduce_sum(x, axis=axis)\n  ysum = tf.reduce_sum(y, axis=axis)\n  xmean = xsum / n\n  ymean = ysum / n\n  xsqsum = tf.reduce_sum( math_ops.squared_difference(x, xmean), axis=axis)\n  ysqsum = tf.reduce_sum( math_ops.squared_difference(y, ymean), axis=axis)\n  cov = tf.reduce_sum( (x - xmean) * (y - ymean), axis=axis)\n  corr = cov / tf.sqrt(xsqsum * ysqsum)\n  sqdif = tf.reduce_sum(tf.squared_difference(x, y), axis=axis) / n / tf.sqrt(ysqsum / n)\n  return tf.convert_to_tensor( K.mean(tf.constant(1.0, dtype=x.dtype) - corr + (0.01 * sqdif)) , dtype=tf.float32 )\n\n\ndef correlationMetric(x, y, axis=-2):\n  x = tf.convert_to_tensor(x)\n  y = math_ops.cast(y, x.dtype)\n  n = tf.cast(tf.shape(x)[axis], x.dtype)\n  xsum = tf.reduce_sum(x, axis=axis)\n  ysum = tf.reduce_sum(y, axis=axis)\n  xmean = xsum / n\n  ymean = ysum / n\n  xvar = tf.reduce_sum(math_ops.squared_difference(x, xmean), axis=axis)\n  yvar = tf.reduce_sum(math_ops.squared_difference(y, ymean), axis=axis)\n  cov = tf.reduce_sum( (x - xmean) * (y - ymean), axis=axis)\n  corr = cov / tf.sqrt(xvar * yvar)\n  return tf.constant(1.0, dtype=x.dtype) - corr\n\ncorrelationMetric(tf.constant([[0.0, 1.0, 2.0]]), tf.constant([[1.0, 3.0, 2.0]]), axis=-1)\ncorrelationMetric(tf.constant([[0.0, 2.0, 1.0]]), tf.constant([[1.0, 3.0, 2.0]]), axis=-1)\ncorrelationMetric(tf.constant([[0.0], [2.0], [1.0]]), tf.constant([[1.0], [3.0], [2.0]]), axis=-2)\n\nmodel.compile(loss='mean_squared_error', optimizer='sgd', metrics=[correlationMetric, 'mean_squared_error'])","metadata":{"id":"GHYkdbB151Vy","execution":{"iopub.status.busy":"2022-04-28T05:00:22.670926Z","iopub.execute_input":"2022-04-28T05:00:22.67127Z","iopub.status.idle":"2022-04-28T05:00:22.710998Z","shell.execute_reply.started":"2022-04-28T05:00:22.671239Z","shell.execute_reply":"2022-04-28T05:00:22.709618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text=bert_preprocess_model([np.array(train_dfsam['anchor'].to_list()), np.array(train_dfsam['target'].to_list())])\ntrain_cat=np.array(train_dfsam[cat_var_list])\nval_text=bert_preprocess_model([np.array(val_df['anchor'].to_list()), np.array(val_df['target'].to_list())])\nval_cat=np.array(val_df[cat_var_list])","metadata":{"id":"fb0wpBjh57R8","execution":{"iopub.status.busy":"2022-04-28T04:56:46.162075Z","iopub.execute_input":"2022-04-28T04:56:46.162873Z","iopub.status.idle":"2022-04-28T04:56:48.58373Z","shell.execute_reply.started":"2022-04-28T04:56:46.162812Z","shell.execute_reply":"2022-04-28T04:56:48.583002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=np.array(pd.to_numeric(train_dfsam['score']))\ny_val=np.array(pd.to_numeric(val_df['score']))","metadata":{"id":"dti-y4rw9hJY","execution":{"iopub.status.busy":"2022-04-28T04:56:48.585081Z","iopub.execute_input":"2022-04-28T04:56:48.585422Z","iopub.status.idle":"2022-04-28T04:56:48.590576Z","shell.execute_reply.started":"2022-04-28T04:56:48.585393Z","shell.execute_reply":"2022-04-28T04:56:48.589383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    # optimizer = keras.optimizers.Adam(lr=5e-5)\n    # model.compile(optimizer=optimizer, loss=[loss, loss])\n    # rmsprop\n    # sgd\n    # model.compile(loss='cosine_similarity', optimizer='sgd', metrics=[correlationMetric, 'accuracy'])","metadata":{"id":"QEBeurUCWNRB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit([train_text, train_cat], y_train, validation_data=([val_text, val_cat], y_val), epochs=6)","metadata":{"id":"WvPogj-S6P00","outputId":"57e983d7-9cf4-497a-a8e8-f78bba391d33","execution":{"iopub.status.busy":"2022-04-28T05:19:48.470359Z","iopub.execute_input":"2022-04-28T05:19:48.470679Z","iopub.status.idle":"2022-04-28T06:02:57.781463Z","shell.execute_reply.started":"2022-04-28T05:19:48.470645Z","shell.execute_reply":"2022-04-28T06:02:57.780352Z"},"trusted":true},"execution_count":null,"outputs":[]}]}