test push

HW01/Test.md

@@ -0,0 +1,3 @@
+# 一些GIT问题
+https://blog.csdn.net/appleyuchi/article/details/102766591
+https://blog.csdn.net/huhao820/article/details/120308442?spm=1001.2101.3001.6661.1&utm_medium=distribute.pc_relevant_t0.none-task-blog-2%7Edefault%7ECTRLIST%7ERate-1-120308442-blog-124432254.pc_relevant_aa&depth_1-utm_source=distribute.pc_relevant_t0.none-task-blog-2%7Edefault%7ECTRLIST%7ERate-1-120308442-blog-124432254.pc_relevant_aa&utm_relevant_index=1

HW01/洪钦敏-HW1/config.py

@@ -0,0 +1,14 @@
+version = 21
+config = {
+    'seed': 1,  # Your seed number, you can pick your lucky number. :)
+    'select_all': False,  # Whether to use all features.
+    'valid_ratio': 0.2,  # validation_size = train_size * valid_ratio
+    'n_epochs': 10000,  # Number of epochs.
+    'batch_size': 100,
+    'learning_rate': 1e-4,
+    'weight_decay': 1e-3,
+    'feature_k': 20,
+    'early_stop': 3000,  # If model has not improved for this many consecutive epochs, stop training.
+    'save_path': './models/model-v%d.ckpt' % version,  # Your model will be saved here.
+    'pred_path': 'pred-v%d.csv' % version  # Your model will be saved here.
+}

HW01/洪钦敏-HW1/dataset.py

@@ -0,0 +1,25 @@
+import torch
+from torch.utils.data import Dataset
+
+
+class COVID19Dataset(Dataset):
+    '''
+    x: Features.
+    y: Targets, if none, do prediction.
+    '''
+
+    def __init__(self, x, y=None):
+        if y is None:
+            self.y = y
+        else:
+            self.y = torch.FloatTensor(y)
+        self.x = torch.FloatTensor(x)
+
+    def __getitem__(self, idx):
+        if self.y is None:
+            return self.x[idx]
+        else:
+            return self.x[idx], self.y[idx]
+
+    def __len__(self):
+        return len(self.x)

HW01/洪钦敏-HW1/learnw.py

@@ -0,0 +1,44 @@
+import numpy as np
+
+w = 1.2
+b = 0.5
+
+dataset = [
+    [0.2, 1],
+    [0.7, 0.25],
+    [0.6, 1.1],
+    [1.2, 5.2],
+    [1.3, 3.1],
+]
+
+origin = np.array(dataset)
+x = origin[:, 0]
+y = origin[:, 1]
+print(f'x:{x} y:{y}')
+# 计算第一次的y
+y1 = w * x + b
+print('y1', y1)
+size = y.size
+mse = ((y1 - y) ** 2).sum() / size
+print(mse)
+# cost函数求导
+
+dw = 2 * (w * (x ** 2).sum() - ((y - b) * x).sum())
+db = 2 * (size * b - ((y - w * x).sum()))
+print(f'dw:{dw} db:{db}')
+num = 0
+lr = 1e-3
+while dw != 0 or db != 0:
+    w = w - lr * dw
+    b = b - lr * db
+    dw = 2 * (w * (x ** 2).sum() - ((y - b) * x).sum())
+    db = 2 * (size * b - ((y - w * x).sum()))
+    num += 1
+    if num > 10000:
+        break
+
+print(num)
+print(w)
+print(b)
+mse = ((w * x + b - y) ** 2).sum() / size
+print(mse)

HW01/洪钦敏-HW1/main.py

@@ -0,0 +1,98 @@
+# Numerical Operations
+import math
+import time
+
+import numpy as np
+
+# Reading/Writing Data
+import pandas as pd
+import os
+import csv
+
+# For Progress Bar
+from tqdm import tqdm
+
+# Pytorch
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader, random_split
+# random_split随机将一个数据集分割成给定长度的不重叠的新数据集。可选择固定生成器以获得可复现的结果
+
+from sklearn.feature_selection import SelectKBest
+from sklearn.feature_selection import f_regression
+# For plotting learning curve
+from torch.utils.tensorboard import SummaryWriter
+
+from utility import *
+from config import config
+from dataset import COVID19Dataset
+from model import trainer, My_Model
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+
+# Choose features you deem useful by modifying the function below.
+def select_feat(train_data, valid_data, test_data, select_all=True, feature_idx=None):
+    '''Selects useful features to perform regression'''
+    y_train, y_valid = train_data[:, -1], valid_data[:, -1]
+    raw_x_train, raw_x_valid, raw_x_test = train_data[:, :-1], valid_data[:, :-1], test_data
+
+    if select_all:
+        feat_idx = list(range(raw_x_train.shape[1]))
+    else:
+        # feat_idx = list(range(raw_x_train.shape[1]))[1:] # TODO: Select suitable feature columns.
+        feat_idx = feature_idx
+        # print(feat_idx)
+
+    return raw_x_train[:, feat_idx], raw_x_valid[:, feat_idx], raw_x_test[:, feat_idx], y_train, y_valid
+
+
+if __name__ == '__main__':
+    # Set seed for reproducibility
+    same_seed(config['seed'])
+
+    features = pd.read_csv('./covid.train.csv')
+    x_data, y_data = features.iloc[:, :-1], features.iloc[:, -1]
+    k=config['feature_k']
+    selector = SelectKBest(score_func=f_regression, k=config['feature_k'])
+    result = selector.fit(x_data, y_data)
+    idx = np.argsort(result.scores_)[::-1]
+    print(x_data.columns[idx[:k]])
+    feature_idx = list(np.sort(idx[:k]))
+    print(feature_idx)
+    time.sleep(3)
+
+    # train_data size: 2699 x 118 (id + 37 states + 16 features x 5 days)
+    # test_data size: 1078 x 117 (without last day's positive rate)
+    train_data, test_data = pd.read_csv('./covid.train.csv').values, pd.read_csv('./covid.test.csv').values
+    train_data, valid_data = train_valid_split(train_data, config['valid_ratio'], config['seed'])
+
+    # Print out the data size.
+    print(f"""train_data size: {train_data.shape} 
+    valid_data size: {valid_data.shape} 
+    test_data size: {test_data.shape}""")
+
+    # Select features
+    x_train, x_valid, x_test, y_train, y_valid = select_feat(train_data, valid_data, test_data, config['select_all'],
+                                                             feature_idx)
+
+    # Print out the number of features.
+    print(f'number of features: {x_train.shape[1]}')
+
+    # init dataset
+    train_dataset = COVID19Dataset(x_train, y_train)
+    valid_dataset = COVID19Dataset(x_valid, y_valid)
+    test_dataset = COVID19Dataset(x_test)
+
+    # Pytorch data loader loads pytorch dataset into batches.
+    train_loader = DataLoader(train_dataset, batch_size=config['batch_size'], shuffle=True, pin_memory=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=config['batch_size'], shuffle=True, pin_memory=True)
+    test_loader = DataLoader(test_dataset, batch_size=config['batch_size'], shuffle=False, pin_memory=True)
+
+    model = My_Model(input_dim=x_train.shape[1]).to(device)  # put your model and data on the same computation device.
+    trainer(train_loader, valid_loader, model, config, device)
+
+    model = My_Model(input_dim=x_train.shape[1]).to(device)
+    model.load_state_dict(torch.load(config['save_path']))
+    preds = predict(test_loader, model, device)
+    save_pred(preds, config['pred_path'])

HW01/洪钦敏-HW1/model.py

@@ -0,0 +1,124 @@
+import math
+import os
+
+import torch
+import torch.nn as nn
+from torch.utils.tensorboard import SummaryWriter
+from tqdm import tqdm
+
+
+# tensorboard --logdir=D:\projects\ML2022-Spring\HW01\洪钦敏-HW1\runs
+class My_Model(nn.Module):
+    def __init__(self, input_dim):
+        super(My_Model, self).__init__()
+        d1 = 64
+        d2 = 16
+        d3 = 8
+        layer = 2
+        # TODO: modify model's structure, be aware of dimensions.
+        if 3 == layer:
+            self.layers = nn.Sequential(
+                # nn.BatchNorm1d(input_dim),
+                nn.Linear(input_dim, d1),
+                nn.ReLU(),
+                # nn.LeakyReLU(0.2),
+                nn.Dropout(0.2),
+                nn.Linear(d1, d2),
+                # nn.ReLU(),
+                nn.LeakyReLU(),
+                # nn.Dropout(0.2),
+                nn.Linear(d2, 1)
+            )
+        elif 2 == layer:
+            self.layers = nn.Sequential(
+                # 重要的改动 https://blog.csdn.net/qq_23262411/article/details/100175943
+                # 然而没啥用貌似
+                # nn.BatchNorm1d(input_dim),
+                nn.Linear(input_dim, d1),
+                # nn.BatchNorm1d(d1),
+                nn.LeakyReLU(),
+                # nn.Dropout(0.2),
+                nn.Linear(d1, 1)
+            )
+        elif 1 == layer:
+            self.layers = nn.Sequential(
+                nn.BatchNorm1d(input_dim),
+                nn.Linear(input_dim, 1)
+            )
+
+    def forward(self, x):
+        x = self.layers(x)
+        x = x.squeeze(1)  # (B, 1) -> (B)
+        return x
+
+
+def trainer(train_loader, valid_loader, model, config, device):
+    criterion = nn.MSELoss(reduction='mean')  # Define your loss function, do not modify this.
+
+    # Define your optimization algorithm.
+    # TODO: Please check https://pytorch.org/docs/stable/optim.html to get more available algorithms.
+    # TODO: L2 regularization (optimizer(weight decay...) or implement by your self).
+    # optimizer = torch.optim.SGD(model.parameters(), lr=config['learning_rate'], momentum=0.9)
+    # optimizer = torch.optim.ASGD(model.parameters(), lr=config['learning_rate'])
+    # optimizer = torch.optim.SGD(model.parameters(), lr=config['learning_rate'], momentum=0.9, weight_decay=config['weight_decay'])
+    optimizer = torch.optim.Adam(model.parameters(), betas=[0.9, 0.999], lr=config['learning_rate'], eps=1e-6,
+                                 weight_decay=config['weight_decay'])
+
+    writer = SummaryWriter()  # Writer of tensoboard.
+
+    if not os.path.isdir('./models'):
+        os.mkdir('./models')  # Create directory of saving models.
+
+    n_epochs, best_loss, step, early_stop_count = config['n_epochs'], math.inf, 0, 0
+
+    for epoch in range(n_epochs):
+        model.train()  # Set your model to train mode.
+        loss_record = []
+
+        # tqdm is a package to visualize your training progress.
+        train_pbar = tqdm(train_loader, position=0, leave=True)
+
+        for x, y in train_pbar:
+            optimizer.zero_grad()  # Set gradient to zero.
+            x, y = x.to(device), y.to(device)  # Move your data to device.
+            pred = model(x)
+            loss = criterion(pred, y)
+            loss.backward()  # Compute gradient(backpropagation).
+            optimizer.step()  # Update parameters.
+            step += 1
+            loss_record.append(loss.detach().item())
+
+            # Display current epoch number and loss on tqdm progress bar.
+            train_pbar.set_description(f'Epoch [{epoch + 1}/{n_epochs}]')
+            train_pbar.set_postfix({'loss': loss.detach().item()})
+
+        mean_train_loss = sum(loss_record) / len(loss_record)
+        writer.add_scalar('Loss/train', mean_train_loss, step)
+
+        model.eval()  # Set your model to evaluation mode.
+        loss_record = []
+        for x, y in valid_loader:
+            x, y = x.to(device), y.to(device)
+            # 验证的时候无需计算梯度下降
+            with torch.no_grad():
+                pred = model(x)
+                loss = criterion(pred, y)
+
+            loss_record.append(loss.item())
+
+        mean_valid_loss = sum(loss_record) / len(loss_record)
+        print(f'Epoch [{epoch + 1}/{n_epochs}]: Train loss: {mean_train_loss:.4f}, Valid loss: {mean_valid_loss:.4f}')
+        writer.add_scalar('Loss/valid', mean_valid_loss, step)
+
+        if mean_valid_loss < best_loss:
+            best_loss = mean_valid_loss
+            torch.save(model.state_dict(), config['save_path'])  # Save your best model
+            print('Saving model with loss {:.3f}...'.format(best_loss))
+            early_stop_count = 0
+        else:
+            early_stop_count += 1
+
+        # 训练一定次数后没有更好的loss 则退出训练
+        if early_stop_count >= config['early_stop']:
+            print('\nModel is not improving, so we halt the training session.')
+            return