Trained the Model

Author: Adammouedden

Data Sets/TLE/propagated_orbits.csv

@@ -15,13 +15,14 @@
 from datetime import datetime, timedelta, timezone
 import numpy as np
 import matplotlib.pyplot as plt
+import math
 
 #Load the dataframe
 df = pd.read_csv("tle_data.csv")
 
 
 
-def propagate_orbit(tle1, tle2, start_time, duration=3600, step=60):
+def propagate_orbit(tle1, tle2, start_time, duration=8100, step=30):
     '''
 
     :param tle1: The first line of the TLE
@@ -42,7 +43,7 @@ def propagate_orbit(tle1, tle2, start_time, duration=3600, step=60):
     #Creates a list of timestamps starting from the start time, incremented of "step" seconds until the defined duration.
     #This represents the time points at which the satellite position will be predicted.
     timestamps = []
-    for i in range(0, duration, step):
+    for i in range(0, duration + step, step):
         timestamps.append(start_time + timedelta(seconds=i))
 
     #We will store the positions and velocities at each time step here
@@ -61,6 +62,11 @@ def propagate_orbit(tle1, tle2, start_time, duration=3600, step=60):
         '''
         e, position, velocity = sat.sgp4(julian_date, fraction)
 
+        if any(np.isnan(position)) or any(np.isnan(velocity)):
+            print(f"Bad propagation at {t} for object: {tle1[2:7]} - skipping satellite")
+            return []
+
+
         if (e == 0):
             #Append the current time stamp, position, and velocity
             results.append({
@@ -72,11 +78,13 @@ def propagate_orbit(tle1, tle2, start_time, duration=3600, step=60):
                 "velocity_y": velocity[1],
                 "velocity_z": velocity[2]
             })
-
-
+        if math.isnan(position[0]) or math.isnan(velocity[0]):
+            print(f"NaN detected at time {t}")
     return results
 
 def propagate_row(row):
+    object_id = row["tle_line1"][2:7]
+    print(f"Propagating satellite {object_id}")
     return propagate_orbit(row["tle_line1"], row["tle_line2"], start_time)
 
 def graph_positions(df):
@@ -165,9 +173,21 @@ def parse_data(propagated_data):
     array = np.array([[sv["position_x"], sv["position_y"], sv["position_z"], sv["velocity_x"], sv["velocity_y"], sv["velocity_z"]] for sv in propagated_data])
     return array
 
-def data_formatting(df):
+def data_formatting(df, duration=8100, steps=30):
     # Process each row into a list of sequences
-    sequences = [parse_data(row["propagated"]) for i, row in df.iterrows()]
+
+    expected_timesteps = int(duration / steps) + 1
+
+    sequences = []
+    #Drop any invalid propagated sequences
+    for i, row in df.iterrows():
+        parsed = parse_data(row["propagated"])
+        if parsed.shape == (expected_timesteps, 6):
+            sequences.append(parsed)
+        else:
+            print(f"Dropping incomplete sequence {i}: shape {parsed.shape}")
+
+
     # Convert to (B, T, F) NumPy array
     data_array = np.array(sequences)  # Shape: (B, T, 6)
 

Data Sets/TLE/tle_propagate.py

@@ -1,9 +1,5 @@
 import torch
 import torch.nn as nn
-import torch.nn.functional as F
-import pandas as pd
-from torch.nn.functional import dropout
-from sklearn.preprocessing import StandardScaler
 
 #Hyperparameters:
 '''
@@ -21,19 +17,6 @@
      
 '''
 
-INPUT_DIM = 7          #The dimensions/neurons for the input layer: time, pos_x, pos_y, pos_z, vel_x, vel_y, vel_z
-EMBED_DIM = 128        #Embedding Dimension for input vectors.
-NUM_HEADS = 8          #Number of attention heads in multi-head attention block
-NUM_LAYERS = 6         #Number of encoder layers
-FEED_FORWARD_DIM = 256 #Size of feedforward layers within the Transformer's MLP
-OUTPUT_DIM = 6         #Predicting the 6 dimensional outputs (the next state vectors)
-SEQ_LENGTH = 10        #Length of the input sequences
-LEARNING_RATE = 0.001  #The learning rate for the optimizer function
-BATCH_SIZE = 32        #Number of sequences per batch
-EPOCHS = 50            #Number of training iterations
-DROPOUT = 0.1          #Overfitting prevention
-#Add another parameter, dropout, if experiencing overfitting
-
 '''
 Embedding Layer: Since our input consists of 6 continuous features:
 (pos_x, pos_y, pos_z, vel_x, vel_y, vel_z) we will project this into a higher-dimensional space using a fully connected
@@ -97,13 +80,10 @@ def forward(self, src):
         src: input data, tensor of shape (batch_size, sequence_length, embedding_dimensions)
         returns: (batch_size, sequence_length, embedding_dimension)
         '''
-        #Transpose for the transformer so that sequence_length comes first in the tensor
-        #x = x.transpose(0,1)
 
         #Pass through transformer encoder for prediction
         encoded_data = self.encoder(src)
 
-        #Transpose back to having batch first in the tensor
         return encoded_data
 '''
 Transformer Decoder
@@ -128,15 +108,12 @@ def forward(self, tgt, memory):
         tgt: a typical name for the input sequence being sent into a decoder, short for target
         memory: (batch, src_sequence_length, embed_dim) - this is output from the encoder
         '''
-        #Transpose to match transformer input (seq_len, batch, embed_dim), optional
-        #tgt = tgt.transpose(0,1)
-        #memory = memory.transpose(0,1)
+        tgt = tgt
+        memory = memory
 
         #Decode!
         out = self.decoder(tgt=tgt, memory=memory)
 
-        #Transpose back, optional
-        #out = out.transpose(0,1)
         return out
 '''
 Output Layer
@@ -156,14 +133,15 @@ def forward(self,x):
 OrbitAI Transformer Model
 '''
 class OrbitAI(nn.Module):
-    def __init__(self, input_dim, embed_dim, output_dim, num_heads, feedforward_dim, num_layers, dropout, seq_len):
+    def __init__(self, input_dim, embed_dim, output_dim, num_heads, feedforward_dim, num_layers, dropout, seq_len, pred_len):
         super(OrbitAI, self).__init__()
 
         self.embedding = InputEmbedding(
             input_dim = input_dim,
             embed_dim = embed_dim
             )
-        self.positional_encoding = LearnedPositionalEncoding(seq_len, embed_dim)
+        self.src_encoded = LearnedPositionalEncoding(seq_len, embed_dim)
+        self.tgt_encoded = LearnedPositionalEncoding(pred_len, embed_dim)
 
         self.encoder = TransformerEncoder(
             embed_dim = embed_dim,
@@ -192,10 +170,10 @@ def forward(self, src, tgt):
         tgt: [batch_size, tgt_seq_len, input_dim] (from the decoder)
         '''
         src_embedded = self.embedding(src)
-        src_encoded = self.positional_encoding(src_embedded)
+        src_encoded = self.src_encoded(src_embedded)
 
         tgt_embedded = self.embedding(tgt)
-        tgt_encoded = self.positional_encoding(tgt_embedded)
+        tgt_encoded = self.tgt_encoded(tgt_embedded)
 
         #Transformer encoder
         memory = self.encoder(src_encoded)
@@ -208,10 +186,6 @@ def forward(self, src, tgt):
 
         return output
 
-
-
-
-
 '''
 Testing the model so far:
 

Data Sets/TLE/training_data.csv

@@ -0,0 +1,77 @@
+from torch.utils.data import DataLoader
+import matplotlib.pyplot as plt
+from OrbitAITransformer import OrbitAI
+from TransformerTraining import OrbitDataset
+import torch
+
+#-----------------------------------------------------------------------------------------------------------------------
+INPUT_DIM = 7          #The dimensions/neurons for the input layer: time, pos_x, pos_y, pos_z, vel_x, vel_y, vel_z
+EMBED_DIM = 128        #Embedding Dimension for input vectors.
+NUM_HEADS = 8          #Number of attention heads in multi-head attention block
+NUM_LAYERS = 6         #Number of encoder layers
+FEED_FORWARD_DIM = 256 #Size of feedforward layers within the Transformer's MLP
+OUTPUT_DIM = 6         #Predicting the 6 dimensional outputs (the next state vectors)
+LEARNING_RATE = 0.00001  #The learning rate for the optimizer function
+BATCH_SIZE = 32        #Number of sequences per batch
+EPOCHS = 50            #Number of training iterations
+DROPOUT = 0.1          #Overfitting prevention
+#Add another parameter, dropout, if experiencing overfitting
+SEQ_LENGTH = 270        #Length of the input sequences, 270 = 8100/30 = PropagationDuration/steps
+PRED_LEN = 2           #Number of sequences we want outputted
+#-----------------------------------------------------------------------------------------------------------------------
+
+#Load Dataset
+dataset = OrbitDataset(csv_path = "training_data.csv", input_len = 270, pred_len = 1)
+dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)
+device= torch.device('cuda')
+
+#Instantiate model
+MODEL = OrbitAI(
+    input_dim = INPUT_DIM,
+    embed_dim = EMBED_DIM,
+    output_dim = OUTPUT_DIM,
+    num_heads = NUM_HEADS,
+    feedforward_dim = FEED_FORWARD_DIM,
+    num_layers = NUM_LAYERS,
+    dropout = DROPOUT,
+    seq_len = SEQ_LENGTH,
+    pred_len = PRED_LEN
+).to(device)
+
+state_dict = torch.load('orbitai_checkpoint.pth',map_location=device)
+MODEL.load_state_dict(state_dict['model_state_dict'])
+
+# Evaluation
+MODEL.eval()
+with torch.no_grad():
+    sample = dataset[0]
+    src = sample['src'].unsqueeze(0).to(device)    # [1, input_len, 7] this is the input sequence we want the model to learn from, historical trajectory data. When src is encoded it becomes memory
+    tgt = sample['tgt'].unsqueeze(0).to(device)    # [1, pred_len, 7] this is the input to the decoder to start generating predictions, the previously known state
+    tgt_y = sample['tgt_y']                        # [pred_len, 6]
+
+    output = MODEL(src, tgt).squeeze(0).cpu().numpy()  # [pred_len, 6]
+    prediction_unscaled = dataset.inverse_transform(output)
+    target_unscaled = dataset.inverse_transform(tgt_y.numpy())
+
+    print("Prediction:\n", prediction_unscaled[:5])  # Check if it's real values
+    print("Target:\n", target_unscaled[:5])
+
+    time = list(range(target_unscaled.shape[0]))
+
+    print("Time length:", len(time))
+    print("Target shape:", target_unscaled.shape)
+    print("Prediction shape:", prediction_unscaled.shape)
+
+
+    # target_unscaled and prediction_unscaled are shape [pred_len, 6]
+
+    for i, label in enumerate(["X", "Y", "Z"]):
+        plt.figure()
+        plt.plot(time, target_unscaled[:, i], label=f"True{label}")
+        plt.plot(time, prediction_unscaled[:, i], label=f"Predicted{label}")
+        plt.xlabel("Timestep")
+        plt.ylabel(f"Position {label} (km)")
+        plt.title(f"{label}-Axis Position: True vs. Predicted")
+        plt.legend()
+        plt.grid(True)
+        plt.show()

src/Models/OrbitAITransformer.py

@@ -9,19 +9,19 @@
 from sklearn.preprocessing import StandardScaler
 import time
 
-
 INPUT_DIM = 7          #The dimensions/neurons for the input layer: time, pos_x, pos_y, pos_z, vel_x, vel_y, vel_z
 EMBED_DIM = 128        #Embedding Dimension for input vectors.
 NUM_HEADS = 8          #Number of attention heads in multi-head attention block
-NUM_LAYERS = 8         #Number of encoder layers
-FEED_FORWARD_DIM = 512 #Size of feedforward layers within the Transformer's MLP
+NUM_LAYERS = 6         #Number of encoder layers
+FEED_FORWARD_DIM = 256 #Size of feedforward layers within the Transformer's MLP
 OUTPUT_DIM = 6         #Predicting the 6 dimensional outputs (the next state vectors)
-SEQ_LENGTH = 10        #Length of the input sequences
-LEARNING_RATE = 0.0001  #The learning rate for the optimizer function
+LEARNING_RATE = 0.00001  #The learning rate for the optimizer function
 BATCH_SIZE = 32        #Number of sequences per batch
-EPOCHS = 100            #Number of training iterations
+EPOCHS = 50            #Number of training iterations
 DROPOUT = 0.1          #Overfitting prevention
 #Add another parameter, dropout, if experiencing overfitting
+SEQ_LENGTH = 270        #Length of the input sequences, 270 = 8100/30 = PropagationDuration/steps
+PRED_LEN = 2           #Number of sequences we want outputted
 
 
 def train(model, dataloader, optimizer, criterion, device):
@@ -33,6 +33,7 @@ def train(model, dataloader, optimizer, criterion, device):
         tgt = batch['tgt'].to(device)
         tgt_y = batch['tgt_y'].to(device)
 
+
         optimizer.zero_grad()
         output = model(src, tgt)
 
@@ -45,7 +46,7 @@ def train(model, dataloader, optimizer, criterion, device):
     return total_loss / len(dataloader)
 
 class OrbitDataset(Dataset):
-    def __init__(self, csv_path, input_len = 20, pred_len = 10):
+    def __init__(self, csv_path, input_len = 270, pred_len = 1):
         super().__init__()
         self.input_len = input_len
         self.pred_len = pred_len
@@ -54,7 +55,11 @@ def __init__(self, csv_path, input_len = 20, pred_len = 10):
 
         #Keep columns in order
         raw_data = df[['time', 'position_x', 'position_y', 'position_z',
-                                'velocity_x', 'velocity_y', 'velocity_z']].values
+                                'velocity_x', 'velocity_y', 'velocity_z']]
+
+        raw_data = raw_data.dropna().reset_index(drop=True)
+        raw_data = raw_data.values
+        
         #Split time and state data
         self.time = raw_data[:, 0] - raw_data[:, 0].min()/(raw_data[:, 0].max() - raw_data[:, 0].min())
         self.time = self.time.reshape(-1,1) #normalize time
@@ -73,7 +78,7 @@ def __len__(self):
         return len(self.data) - (self.input_len + self.pred_len) + 1
 
     def __getitem__(self, idx):
-        #Encoder input (10 steps)
+        #Encoder input
         src = self.data[idx : idx + self.input_len]
 
         #Decoder input (use the last state from src then roll forward)
@@ -98,8 +103,6 @@ def inverse_transform(self, prediction):
         #Reverse scaling of predicted state vectors, un-normalized them
         return self.scaler.inverse_transform(prediction)
 
-
-
 '''
 #Sanity check
 batch = next(iter(dataloader))
@@ -114,7 +117,7 @@ def inverse_transform(self, prediction):
 device = torch.device('cuda')
 
 #Load Dataset
-dataset = OrbitDataset(csv_path = "training_data.csv", input_len = 10, pred_len = 5)
+dataset = OrbitDataset(csv_path = "training_data.csv", input_len = 270, pred_len = 1)
 dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)
 
 #Instantiate model
@@ -126,7 +129,8 @@ def inverse_transform(self, prediction):
     feedforward_dim = FEED_FORWARD_DIM,
     num_layers = NUM_LAYERS,
     dropout = DROPOUT,
-    seq_len = SEQ_LENGTH
+    seq_len = SEQ_LENGTH,
+    pred_len = PRED_LEN
 ).to(device)
 
 #Select loss function
@@ -152,18 +156,4 @@ def inverse_transform(self, prediction):
 
 print("Model and Optimizer saved")
 
-# Evaluation
-MODEL.eval()
-with torch.no_grad():
-    sample = dataset[0]
-    src = sample['src'].unsqueeze(0).to(device)    # [1, input_len, 7]
-    tgt = sample['tgt'].unsqueeze(0).to(device)    # [1, pred_len, 7]
-    tgt_y = sample['tgt_y']                        # [pred_len, 6]
-
-    output = MODEL(src, tgt).squeeze(0).cpu().numpy()  # [pred_len, 6]
-    prediction_unscaled = dataset.inverse_transform(output)
-    target_unscaled = dataset.inverse_transform(tgt_y.numpy())
-
-    print("Prediction (unscaled):\n", prediction_unscaled)
-    print("Target (unscaled):\n", target_unscaled)