WANGSHUO222 HW1,HW2,HW3,HW4

HW01/WANGSHUO_HW1.py

@@ -0,0 +1,246 @@
+# Numerical Operations
+import math
+import numpy as np
+
+# # Reading/Writing Data
+import pandas as pd
+import os
+import csv
+#
+# # For Progress Bar
+from tqdm import tqdm
+
+# Pytorch
+import torch
+import torch.nn as nn
+import torch.utils.data
+import torchvision
+from torch import nn
+from torch.utils.data import Dataset, DataLoader, random_split
+
+# For plotting learning curve
+from torch.utils.tensorboard import SummaryWriter
+
+    # https://drive.google.com/uc?export=download&id=1kLSW_-cW2Huj7bh84YTdimGBOJaODiOS
+    # https://drive.google.com/uc?export=download&id=1iiI5qROrAhZn-o4FPqsE97bMzDEFvIdg
+# !gdown --id '1kLSW_-cW2Huj7bh84YTdimGBOJaODiOS' --output covid.train.csv
+# !gdown --id '1iiI5qROrAhZn-o4FPqsE97bMzDEFvIdg' --output covid.test.csv
+
+def same_seed(seed):
+    '''Fixes random number generator seeds for reproducibility.'''
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+
+def train_valid_split(data_set, valid_ratio, seed):
+    '''Split provided training data into training set and validation set'''
+    valid_set_size = int(valid_ratio * len(data_set))
+    train_set_size = len(data_set) - valid_set_size
+    train_set, valid_set = random_split(data_set, [train_set_size, valid_set_size],
+                                        generator=torch.Generator().manual_seed(seed))
+    return np.array(train_set), np.array(valid_set)
+
+
+def predict(test_loader, model, device):
+    model.eval()  # Set your model to evaluation mode.
+    preds = []
+    for x in tqdm(test_loader):
+        x = x.to(device)
+        with torch.no_grad():
+            pred = model(x)
+            preds.append(pred.detach().cpu())
+    preds = torch.cat(preds, dim=0).numpy()
+    return preds
+
+
+class COVID19Dataset(Dataset):
+    '''
+    x: Features.
+    y: Targets, if none, do prediction.
+    '''
+
+    def __init__(self, x, y=None):
+        if y is None:
+            self.y = y
+        else:
+            self.y = torch.FloatTensor(y)
+        self.x = torch.FloatTensor(x)
+
+    def __getitem__(self, idx):
+        if self.y is None:
+            return self.x[idx]
+        else:
+            return self.x[idx], self.y[idx]
+
+    def __len__(self):
+        return len(self.x)
+
+
+class My_Model(nn.Module):
+    def __init__(self, input_dim):
+        super(My_Model, self).__init__()
+        # TODO: modify model's structure, be aware of dimensions.
+        self.layers = nn.Sequential(
+            nn.Linear(input_dim, 64),
+            nn.ReLU(),
+            nn.Linear(64, 16),
+            nn.ReLU(),
+            nn.Linear(16, 1)
+        )
+
+    def forward(self, x):
+        x = self.layers(x)
+        x = x.squeeze(1)  # (B, 1) -> (B)
+        return x
+
+
+def select_feat(train_data, valid_data, test_data, select_all=True):
+    '''Selects useful features to perform regression'''
+    y_train, y_valid = train_data[:, -1], valid_data[:, -1]
+    raw_x_train, raw_x_valid, raw_x_test = train_data[:, :-1], valid_data[:, :-1], test_data
+
+    if select_all:
+        feat_idx = list(range(raw_x_train.shape[1]))
+    else:
+        # feat_idx = [0, 1, 2, 3, 4]  # TODO: Select suitable feature columns.
+        # 相关性大于0.8的特征
+        feat_idx = list(range(1, 38)) + [38, 39, 40, 41, 53, 54, 55, 56, 57, 69, 70, 71, 72, 73, 85, 86, 87, 88, 89, 101, 102, 103, 104, 105]
+        # feat_idx = list(range(1, 38)) + [53, 69, 85, 101]
+
+    return raw_x_train[:, feat_idx], raw_x_valid[:, feat_idx], raw_x_test[:, feat_idx], y_train, y_valid
+
+
+def trainer(train_loader, valid_loader, model, config, device):
+    criterion = nn.MSELoss(reduction='mean')  # Define your loss function, do not modify this.
+
+    # Define your optimization algorithm.
+    # TODO: Please check https://pytorch.org/docs/stable/optim.html to get more available algorithms.
+    # TODO: L2 regularization (optimizer(weight decay...) or implement by your self).
+    # optimizer = torch.optim.SGD(model.parameters(), lr=config['learning_rate'], momentum=0.9)
+    # optimizer = torch.optim.AdamW(model.parameters(), lr=config['learning_rate'], weight_decay=0.08)
+    optimizer = torch.optim.Adam(model.parameters(), lr=config['learning_rate'] * 100, weight_decay=1e-3)
+
+    writer = SummaryWriter()  # Writer of tensoboard.
+
+    if not os.path.isdir('./models'):
+        os.mkdir('./models')  # Create directory of saving models.
+
+    n_epochs, best_loss, step, early_stop_count = config['n_epochs'], math.inf, 0, 0
+
+    for epoch in range(n_epochs):
+        model.train()  # Set your model to train mode.
+        loss_record = []
+
+        # tqdm is a package to visualize your training progress.
+        train_pbar = tqdm(train_loader, position=0, leave=True)
+
+        for x, y in train_pbar:
+            optimizer.zero_grad()  # Set gradient to zero.
+            x, y = x.to(device), y.to(device)  # Move your data to device.
+            pred = model(x)
+            loss = criterion(pred, y)
+            loss.backward()  # Compute gradient(backpropagation).
+            optimizer.step()  # Update parameters.
+            step += 1
+            loss_record.append(loss.detach().item())
+
+            # Display current epoch number and loss on tqdm progress bar.
+            train_pbar.set_description(f'Epoch [{epoch + 1}/{n_epochs}]')
+            train_pbar.set_postfix({'loss': loss.detach().item()})
+
+        mean_train_loss = sum(loss_record) / len(loss_record)
+        writer.add_scalar('Loss/train', mean_train_loss, step)
+
+        model.eval()  # Set your model to evaluation mode.
+        loss_record = []
+        for x, y in valid_loader:
+            x, y = x.to(device), y.to(device)
+            with torch.no_grad():
+                pred = model(x)
+                loss = criterion(pred, y)
+
+            loss_record.append(loss.item())
+
+        mean_valid_loss = sum(loss_record) / len(loss_record)
+        print(f'Epoch [{epoch + 1}/{n_epochs}]: Train loss: {mean_train_loss:.4f}, Valid loss: {mean_valid_loss:.4f}')
+        writer.add_scalar('Loss/valid', mean_valid_loss, step)
+
+        if mean_valid_loss < best_loss:
+            best_loss = mean_valid_loss
+            torch.save(model.state_dict(), config['save_path'])  # Save your best model
+            print('Saving model with loss {:.3f}...'.format(best_loss))
+            early_stop_count = 0
+        else:
+            early_stop_count += 1
+
+        if early_stop_count >= config['early_stop']:
+            print('\nModel is not improving, so we halt the training session.')
+            return
+
+
+def save_pred(preds, file):
+    ''' Save predictions to specified file '''
+    with open(file, 'w') as fp:
+        writer = csv.writer(fp)
+        writer.writerow(['id', 'tested_positive'])
+        for i, p in enumerate(preds):
+            writer.writerow([i, p])
+
+
+# 按间距中的绿色按钮以运行脚本。
+if __name__ == '__main__':
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    config = {
+        'seed': 617,  # 制定模型的随即种子以保证模型的可恢复性。Your seed number, you can pick your lucky number. :)
+        'select_all': False,  # Whether to use all features.
+        'valid_ratio': 0.2,  # validation_size = train_size * valid_ratio
+        'n_epochs': 3000,  # Number of epochs.
+        'batch_size': 128,
+        'learning_rate': 1e-5,
+        'early_stop': 400,
+        # If model has not improved for this many consecutive epochs, stop training. 任一时刻连续400次没有模型训练降低loss，就会提前停止。
+        'save_path': './models/model.ckpt'  # Your model will be saved here.
+    }
+
+    # Set seed for reproducibility
+    same_seed(config['seed'])
+
+    # train_data size: 2699 x 118 (id + 37 states + 16 features x 5 days)
+    # test_data size: 1078 x 117 (without last day's positive rate)
+    train_data, test_data = pd.read_csv('./covid.train_new.csv').values, pd.read_csv('./covid.test_un.csv').values
+    train_data, valid_data = train_valid_split(train_data, config['valid_ratio'], config['seed'])  # 按照k折交叉验证法分成训练集和验证集
+
+    # Print out the data size.
+    print(f"""train_data size: {train_data.shape}
+    valid_data size: {valid_data.shape}
+    test_data size: {test_data.shape}""")
+
+    # Select features
+    x_train, x_valid, x_test, y_train, y_valid = select_feat(train_data, valid_data, test_data, config['select_all'])
+
+    # Print out the number of features.
+    print(f'number of features: {x_train.shape[1]}')
+
+    train_dataset, valid_dataset, test_dataset = COVID19Dataset(x_train, y_train), \
+                                                 COVID19Dataset(x_valid, y_valid), \
+                                                 COVID19Dataset(x_test)
+
+    # 用统一的Pytorch加载器包装待处理数据 Pytorch data loader loads pytorch dataset into batches.
+    train_loader = DataLoader(train_dataset, batch_size=config['batch_size'], shuffle=True, pin_memory=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=config['batch_size'], shuffle=True, pin_memory=True)
+    test_loader = DataLoader(test_dataset, batch_size=config['batch_size'], shuffle=False, pin_memory=True)
+
+    model = My_Model(input_dim=x_train.shape[1]).to(device)  # put your model and data on the same computation device.
+    trainer(train_loader, valid_loader, model, config, device)
+
+    # %reload_ext tensorboard
+    # %tensorboard --logdir=./runs/
+
+    model = My_Model(input_dim=x_train.shape[1]).to(device)
+    model.load_state_dict(torch.load(config['save_path']))  # 加载你保存好的best model
+    preds = predict(test_loader, model, device)
+    save_pred(preds, 'pred.csv')