Skip to content

Commit

Permalink
changed bert_sentiment & bert_rnn_sep to use classes for training
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ignatiocalvin
Ignatiocalvin committed Jun 7, 2024
1 parent 3f43349 commit 49d56fa
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 174 deletions.
Binary file modified __pycache__/models.cpython-39.pyc
View file
Binary file not shown.
Binary file modified __pycache__/utils.cpython-39.pyc
View file
Binary file not shown.
Empty file added bert_MA.ipynb
View file
Empty file.
88 changes: 31 additions & 57 deletions bert_rnn_sep.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,16 @@
"execution_count": 2,
"id": "2cc49abb-d4bd-422e-98a4-b10c91bfa35c",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/ignatiocalvin/anaconda3/envs/dowJones/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from models import StockPredictor\n",
"\n",
Expand All @@ -75,92 +84,57 @@
"hidden_dim = 128\n",
"num_layers = 2\n",
"\n",
"model = StockPredictor(embedding_dim, price_dim, hidden_dim, num_layers)\n"
"model = StockPredictor(embedding_dim, price_dim, hidden_dim, num_layers)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "be347a74-7360-4836-9204-74f19db79dad",
"id": "85838d3d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/ignatiocalvin/anaconda3/envs/dowJones/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1, Loss: 178201392.0, MAE: 13349.209414793073\n",
"Epoch 2, Loss: 146288288.0, MAE: 12094.969532826446\n",
"Epoch 3, Loss: 129430936.0, MAE: 11376.771774101826\n",
"Epoch 4, Loss: 61263784.0, MAE: 7827.118499166855\n",
"Epoch 5, Loss: 35276108.0, MAE: 5939.36932678883\n",
"Epoch 6, Loss: 9780979.0, MAE: 3127.455675145533\n",
"Epoch 7, Loss: 11902432.0, MAE: 3449.9901449134604\n",
"Epoch 8, Loss: 5983936.0, MAE: 2446.2084947935246\n",
"Epoch 9, Loss: 6555874.0, MAE: 2560.4441021041644\n",
"Epoch 10, Loss: 6831317.5, MAE: 2613.6789206021463\n"
"Epoch 1, Loss: 160156768.0, MAE: 12655.30592281356\n",
"Epoch 2, Loss: 164208880.0, MAE: 12814.401273567173\n",
"Epoch 3, Loss: 126178272.0, MAE: 11232.910219529042\n",
"Epoch 4, Loss: 46271344.0, MAE: 6802.304315450758\n",
"Epoch 5, Loss: 17988900.0, MAE: 4241.332337839137\n",
"Epoch 6, Loss: 6537196.5, MAE: 2556.794184129806\n",
"Epoch 7, Loss: 7154466.0, MAE: 2674.783355713131\n",
"Epoch 8, Loss: 11508705.0, MAE: 3392.448230997785\n",
"Epoch 9, Loss: 6419246.0, MAE: 2533.6230974633936\n",
"Epoch 10, Loss: 12088799.0, MAE: 3476.8950228616336\n"
]
}
],
"source": [
"from torch.utils.data import DataLoader, TensorDataset\n",
"import torch.optim as optim\n",
"from models import BertRnnTrainer\n",
"\n",
"\n",
"# Training parameters\n",
"num_epochs = 10\n",
"learning_rate = 0.001\n",
"\n",
"# Loss and optimizer\n",
"criterion = nn.MSELoss()\n",
"optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
"\n",
"# Training loop\n",
"model.train()\n",
"for epoch in range(num_epochs):\n",
" for batch_bert, batch_price, batch_y in dataloader_train:\n",
" optimizer.zero_grad()\n",
" outputs = model(batch_bert, batch_price.unsqueeze(-1))\n",
" loss = criterion(outputs.squeeze(), batch_y)\n",
" loss.backward()\n",
" optimizer.step()\n",
" print(f\"Epoch {epoch+1}, Loss: {loss.item()}, MAE: {loss.item()**0.5}\")\n"
"# Assuming you have a model instance 'model' and DataLoaders 'dataloader_train' and 'dataloader_test'\n",
"trainer = BertRnnTrainer(model, dataloader_train, dataloader_test, num_epochs=10, learning_rate=0.001)\n",
"trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "60fddd7f-00e0-44b5-89c3-831ac5e29aca",
"execution_count": 4,
"id": "928d431c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test MSE: tensor(23406206.)\n",
"Test MAE: tensor(4837.9961)\n"
"Test MSE: tensor(23333504.)\n",
"Test MAE: tensor(4830.4766)\n"
]
}
],
"source": [
"import torchmetrics\n",
"\n",
"mse = torchmetrics.MeanSquaredError()\n",
"model.eval()\n",
"with torch.no_grad():\n",
" for batch_bert, batch_price, batch_y in dataloader_test:\n",
" outputs = model(batch_bert, batch_price.unsqueeze(-1))\n",
" mse(outputs.squeeze(), batch_y)\n",
"\n",
"print(\"Test MSE: \", mse.compute())\n",
"print(\"Test MAE: \", mse.compute()**0.5) "
"trainer.test()"
]
}
],
Expand Down
147 changes: 32 additions & 115 deletions bert_sentiment.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 43,
"execution_count": 1,
"id": "3a8429b5-abda-4859-9d0c-93afa09af9f4",
"metadata": {},
"outputs": [],
Expand All @@ -12,6 +12,7 @@
"import pandas as pd\n",
"import torch.nn as nn\n",
"from torch.utils.data import TensorDataset, DataLoader\n",
"from utils import create_sequences_sent\n",
"\n",
"# import all data\n",
"df = pd.read_csv('data/upload_DJIA_table.csv', parse_dates=['Date'], index_col='Date')\n",
Expand All @@ -27,31 +28,13 @@
"bert_embeddings_train = bert_embeddings[:train_data.shape[0]].reshape(-1, 768)\n",
"bert_embeddings_test = bert_embeddings[train_data.shape[0]:].reshape(-1, 768)\n",
"\n",
"\n",
"def create_sequences(df, seq_length, bert_emb):\n",
" xemb, x_pr, x_sent, ys = [], [], [], []\n",
" # Iterate over data indices\n",
" for i in range(len(df) - seq_length):\n",
" \t# Define inputs\n",
" xemb.append(bert_emb[i:i+seq_length])\n",
" x_pr.append(df.iloc[i:i+seq_length, 0].values.reshape(-1, 1))\n",
" x_sent.append(fingpt_sentiments[i:i+seq_length])\n",
" \n",
" # xemb = np.concatenate((xemb, fingpt_sentiments[i:i+seq_length].reshape(-1, 1)), axis=1)\n",
" # Define target\n",
" y = df.iloc[i+seq_length, 0]\n",
" # xs.append(x)\n",
" ys.append(y)\n",
" return np.array(xemb), np.array(x_pr).squeeze(), np.array(x_sent), np.array(ys)\n",
" \n",
"\n",
"X_train_emb, X_train_pr, X_train_sent, y_train = create_sequences(train_data, 60, bert_embeddings_train)\n",
"X_test_emb, X_test_pr, X_test_sent, y_test = create_sequences(test_data, 60, bert_embeddings_test)"
"X_train_emb, X_train_pr, X_train_sent, y_train = create_sequences_sent(train_data, 60, bert_embeddings_train, fingpt_sentiments)\n",
"X_test_emb, X_test_pr, X_test_sent, y_test = create_sequences_sent(test_data, 60, bert_embeddings_test, fingpt_sentiments)"
]
},
{
"cell_type": "code",
"execution_count": 44,
"execution_count": 2,
"id": "d0a527a2-935a-4809-a162-83d5c395059b",
"metadata": {},
"outputs": [
Expand All @@ -61,7 +44,7 @@
"((1551, 60), (318, 60))"
]
},
"execution_count": 44,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -72,7 +55,7 @@
},
{
"cell_type": "code",
"execution_count": 45,
"execution_count": 3,
"id": "fda13211-51f3-4192-aefc-846657dbace9",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -109,138 +92,72 @@
},
{
"cell_type": "code",
"execution_count": 50,
"execution_count": 10,
"id": "2cc49abb-d4bd-422e-98a4-b10c91bfa35c",
"metadata": {},
"outputs": [],
"source": [
"import torch.nn as nn\n",
"\n",
"class StockPredictor(nn.Module):\n",
" def __init__(self, embedding_dim, price_dim, sent_dim, hidden_dim, num_layers):\n",
" super(StockPredictor, self).__init__()\n",
" self.lstm_bert = nn.LSTM(embedding_dim, hidden_dim, num_layers, batch_first=True)\n",
" self.lstm_price = nn.LSTM(price_dim, hidden_dim, num_layers, batch_first=True)\n",
" self.lstm_sent = nn.LSTM(sent_dim, hidden_dim, num_layers, batch_first=True)\n",
" # self.fc = nn.Linear(hidden_dim * 2, 1)\n",
" self.head = nn.Sequential(\n",
" nn.Linear(hidden_dim*3, 64),\n",
" nn.ReLU(),\n",
" nn.Linear(64, 32),\n",
" nn.ReLU(),\n",
" nn.Linear(32, 1)\n",
" )\n",
" \n",
" def forward(self, bert_x, price_x, sent_x):\n",
" lstm_out_bert, _ = self.lstm_bert(bert_x)\n",
" lstm_out_price, _ = self.lstm_price(price_x)\n",
" lstm_out_sent, _ = self.lstm_sent(sent_x)\n",
" # Only consider the output of the last LSTM cell\n",
" last_out_bert = lstm_out_bert[:, -1, :]\n",
" last_out_price = lstm_out_price[:, -1, :]\n",
" last_out_sent = lstm_out_sent[:, -1, :]\n",
" \n",
" # Concatenate the outputs of both LSTMs\n",
" combined_out = torch.cat((last_out_bert, last_out_price, last_out_sent), dim=1)\n",
" output = self.head(combined_out)\n",
" return output\n",
"from models import StockPredictorSent\n",
"\n",
"# Model parameters\n",
"embedding_dim = 768 # Size of BERT embeddings\n",
"price_dim = 1 # Each stock price is a single number\n",
"sent_dim = 1 # Each sentiment is a single number\n",
"\n",
"hidden_dim = 128\n",
"num_layers = 2\n",
"\n",
"model = StockPredictor(embedding_dim, price_dim, sent_dim, hidden_dim, num_layers)\n"
"model = StockPredictorSent(embedding_dim, price_dim, sent_dim, hidden_dim, num_layers)"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "be347a74-7360-4836-9204-74f19db79dad",
"execution_count": 11,
"id": "27cbb8a2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1, Loss: 144917312.0, MAE: 12038.16065684455\n",
"Epoch 2, Loss: 162123424.0, MAE: 12732.76969084103\n",
"Epoch 3, Loss: 116569328.0, MAE: 10796.727652395424\n",
"Epoch 4, Loss: 33920120.0, MAE: 5824.0982134576\n",
"Epoch 5, Loss: 13177386.0, MAE: 3630.0669415315197\n",
"Epoch 6, Loss: 7766516.5, MAE: 2786.8470535714728\n",
"Epoch 7, Loss: 8222755.5, MAE: 2867.534742596853\n",
"Epoch 8, Loss: 7412333.5, MAE: 2722.5601003467305\n",
"Epoch 9, Loss: 4933209.0, MAE: 2221.082844020006\n",
"Epoch 10, Loss: 4819522.0, MAE: 2195.3409757939653\n"
"Epoch 1, Loss: 126368232.0, MAE: 11241.362550865442\n",
"Epoch 2, Loss: 141721200.0, MAE: 11904.671352036561\n",
"Epoch 3, Loss: 139982544.0, MAE: 11831.421892570648\n",
"Epoch 4, Loss: 82866856.0, MAE: 9103.123420013595\n",
"Epoch 5, Loss: 35907120.0, MAE: 5992.255001249529\n",
"Epoch 6, Loss: 16482364.0, MAE: 4059.847780397684\n",
"Epoch 7, Loss: 7530426.0, MAE: 2744.1621672197143\n",
"Epoch 8, Loss: 7355697.0, MAE: 2712.138823880518\n",
"Epoch 9, Loss: 8914146.0, MAE: 2985.6567116800284\n",
"Epoch 10, Loss: 8588848.0, MAE: 2930.673642697187\n"
]
}
],
"source": [
"from torch.utils.data import DataLoader, TensorDataset\n",
"import torch.optim as optim\n",
"\n",
"\n",
"# Training parameters\n",
"num_epochs = 10\n",
"learning_rate = 0.001\n",
"\n",
"# Loss and optimizer\n",
"criterion = nn.MSELoss()\n",
"optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
"from models import BertSentimentTrainer\n",
"\n",
"# Training loop\n",
"model.train()\n",
"for epoch in range(num_epochs):\n",
" for batch_bert, batch_price, batch_sent, batch_y in dataloader_train:\n",
" optimizer.zero_grad()\n",
" outputs = model(batch_bert, batch_price.unsqueeze(-1), batch_sent.unsqueeze(-1))\n",
" loss = criterion(outputs.squeeze(), batch_y)\n",
" loss.backward()\n",
" optimizer.step()\n",
" print(f\"Epoch {epoch+1}, Loss: {loss.item()}, MAE: {loss.item()**0.5}\")\n"
"# Assuming you have a model instance 'model' and DataLoaders 'dataloader_train' and 'dataloader_test'\n",
"trainer = BertSentimentTrainer(model, dataloader_train, dataloader_test, num_epochs=10, learning_rate=0.001)\n",
"trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "60fddd7f-00e0-44b5-89c3-831ac5e29aca",
"execution_count": 12,
"id": "0820c34d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test MSE: tensor(23709278.)\n",
"Test MAE: tensor(4869.2173)\n"
"Test MSE: tensor(23712514.)\n",
"Test MAE: tensor(4869.5498)\n"
]
}
],
"source": [
"import torchmetrics\n",
"\n",
"\n",
"mse = torchmetrics.MeanSquaredError()\n",
"model.eval()\n",
"with torch.no_grad():\n",
" for batch_bert, batch_price, batch_sent, batch_y in dataloader_test:\n",
" outputs = model(batch_bert, batch_price.unsqueeze(-1), batch_sent.unsqueeze(-1))\n",
" mse(outputs.squeeze(), batch_y)\n",
"\n",
"print(\"Test MSE: \", mse.compute())\n",
"print(\"Test MAE: \", mse.compute()**0.5) "
"trainer.test()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d4196348-5179-4f6a-8ea1-447cee513531",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand All @@ -259,7 +176,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
"version": "3.9.18"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit 49d56fa

Please sign in to comment.