Merge pull request #331 from nasaharvest/Zambia-2019

Zambia 2019
nasaharvest · Jun 22, 2023 · 76243c5 · 76243c5
2 parents cc520ee + 916e6cb
commit 76243c5
Show file tree

Hide file tree

Showing 9 changed files with 155 additions and 45 deletions.
diff --git a/data/all_dataset_params.json b/data/all_dataset_params.json
@@ -49,6 +49,58 @@
             12
         ]
     },
+    "geowiki_landcover_2017,Kenya,Mali,Mali_lower_CEO_2019,Mali_upper_CEO_2019,Togo,Rwanda,Uganda,digitalearthafrica_sahel,Ethiopia_Tigray_2020,Ethiopia_Tigray_2021,Ethiopia_Bure_Jimma_2019,Ethiopia_Bure_Jimma_2020,Argentina_Buenos_Aires,Malawi_CEO_2020,Malawi_CEO_2019,Malawi_FAO,Zambia_CEO_2019,Tanzania_CEO_2019,Namibia_CEO_2020,Sudan_Blue_Nile_CEO_2019,Hawaii_CEO_2020_November_2022": {
+        "normalizing_dict": {
+            "mean": [
+                -11.327710250226808,
+                -18.283221277270346,
+                1450.819834520752,
+                1393.9268881287114,
+                1401.8089939875538,
+                1642.452605965862,
+                2393.883585822518,
+                2762.860224846127,
+                2655.367583621882,
+                3015.6448988512634,
+                845.0112062171896,
+                2345.184937131625,
+                1526.7973954710592,
+                292.5398906098656,
+                0.003541910725533651,
+                845.630116831681,
+                6.131584729268951,
+                0.3464175321740995
+            ],
+            "std": [
+                4.281301132983248,
+                5.2241434341791475,
+                1156.6583957423788,
+                1098.299586107085,
+                1290.4576417134838,
+                1240.6961147822872,
+                1172.7952064054186,
+                1222.8147544315573,
+                1167.5590850234344,
+                1234.4607232693845,
+                722.5235957043584,
+                1075.0021760072136,
+                940.5344835399064,
+                16.425529765676114,
+                0.004164601616655484,
+                724.9284023714741,
+                7.651566162813163,
+                0.23659229629318967
+            ]
+        },
+        "train_num_timesteps": [
+            12,
+            10,
+            5
+        ],
+        "val_num_timesteps": [
+            12
+        ]
+    },
     "geowiki_landcover_2017,Kenya,Mali,Mali_lower_CEO_2019,Mali_upper_CEO_2019,Togo,Rwanda,Uganda,digitalearthafrica_sahel,Ethiopia_Tigray_2020,Ethiopia_Tigray_2021,Ethiopia_Bure_Jimma_2019,Ethiopia_Bure_Jimma_2020,Argentina_Buenos_Aires,Malawi_CEO_2020,Malawi_CEO_2019,Malawi_FAO,Zambia_CEO_2019,Tanzania_CEO_2019,Namibia_CEO_2020,Sudan_Blue_Nile_CEO_2019,Hawaii_CEO_2020_September_2022": {
         "normalizing_dict": {
             "mean": [
@@ -150,6 +202,58 @@
             12
         ]
     },
+    "geowiki_landcover_2017,Kenya,Mali,Mali_lower_CEO_2019,Mali_upper_CEO_2019,Togo,Rwanda,Uganda,open_buildings,digitalearthafrica_eastern,digitalearthafrica_sahel,Ethiopia,Ethiopia_Tigray_2020,Ethiopia_Tigray_2021,Ethiopia_Bure_Jimma_2019,Ethiopia_Bure_Jimma_2020,Argentina_Buenos_Aires,Malawi_CEO_2020,Malawi_CEO_2019,Malawi_FAO,Malawi_FAO_corrected,Zambia_CEO_2019,Tanzania_CEO_2019,Namibia_corrective_labels_2020,Malawi_corrected,Namibia_CEO_2020,Namibia_WFP,Sudan_Blue_Nile_CEO_2019,Hawaii_CEO_2020,HawaiiAgriculturalLandUse2020,KenyaCEO2019,HawaiiCorrective2020,HawaiiCorrectiveGuided2020,MalawiCorrectiveLabels2020,NamibiaFieldBoundary2022,SudanBlueNileCEO2020,SudanBlueNileCorrectiveLabels2019,EthiopiaTigrayCorrective2020,SudanAlGadarefCEO2019,MaliStratifiedCEO2019,SudanAlGadarefCEO2020_November_2022": {
+        "normalizing_dict": {
+            "mean": [
+                -11.439365638877627,
+                -18.606545975998483,
+                1445.183380298818,
+                1418.0857114225628,
+                1465.0864209583292,
+                1700.8462569343772,
+                2446.6537237020307,
+                2825.049998348732,
+                2721.3175918433485,
+                3097.5833672632098,
+                861.1028849845957,
+                2617.600881992651,
+                1782.5853367106095,
+                290.4510318346573,
+                0.003393136822053397,
+                882.9309345196244,
+                5.475135830795568,
+                0.33345542853371174
+            ],
+            "std": [
+                4.181625916423944,
+                5.1313723174266155,
+                978.995221274165,
+                953.4501253327929,
+                1161.152380432254,
+                1112.7187311944313,
+                1046.7482325340723,
+                1110.1392328877532,
+                1074.9588374768962,
+                1145.2968872218312,
+                671.1295298987673,
+                1279.9817959868135,
+                1188.9502877345046,
+                34.91501562567926,
+                0.004325618556478736,
+                630.6249894156837,
+                6.977859068348178,
+                0.22621357797302222
+            ]
+        },
+        "train_num_timesteps": [
+            12,
+            5,
+            10
+        ],
+        "val_num_timesteps": [
+            12
+        ]
+    },
     "geowiki_landcover_2017,Kenya,Mali,Mali_lower_CEO_2019,Mali_upper_CEO_2019,Togo,Rwanda,Uganda,open_buildings,digitalearthafrica_eastern,digitalearthafrica_sahel,Ethiopia,Ethiopia_Tigray_2020,Ethiopia_Tigray_2021,Ethiopia_Bure_Jimma_2019,Ethiopia_Bure_Jimma_2020,Argentina_Buenos_Aires,Malawi_CEO_2020,Malawi_CEO_2019,Malawi_FAO,Malawi_FAO_corrected,Zambia_CEO_2019,Tanzania_CEO_2019,Namibia_corrective_labels_2020,Malawi_corrected,Namibia_CEO_2020,Namibia_WFP,Sudan_Blue_Nile_CEO_2019,Hawaii_CEO_2020,HawaiiAgriculturalLandUse2020,KenyaCEO2019,HawaiiCorrective2020,HawaiiCorrectiveGuided2020,MalawiCorrectiveLabels2020,NamibiaFieldBoundary2022,SudanBlueNileCEO2020,SudanBlueNileCorrectiveLabels2019,EthiopiaTigrayCorrective2020_February_2022": {
         "normalizing_dict": {
             "mean": [

diff --git a/data/datasets.dvc b/data/datasets.dvc
@@ -1,6 +1,6 @@
 outs:
-- md5: a4be1517e3dd9029b82628fd0c587961.dir
-  size: 695317346
+- md5: a3e7e683a3d60682cfb6827a0e344eb1.dir
+  size: 695310613
   nfiles: 43
   path: datasets
   hash: md5
diff --git a/data/models.dvc b/data/models.dvc
@@ -1,6 +1,6 @@
 outs:
-- md5: 8c353c5b7ca65f640a3a6b3c85492134.dir
-  size: 47222896
-  nfiles: 32
+- md5: 67781074f8a00b8d8799796196db5990.dir
+  size: 57927593
+  nfiles: 46
   path: models
   hash: md5
diff --git a/data/models.json b/data/models.json
@@ -238,37 +238,37 @@
         }
     },
     "Zambia_2019": {
-        "params": "https://wandb.ai/nasa-harvest/crop-mask/runs/bfmypvwy",
+        "params": "https://wandb.ai/nasa-harvest/crop-mask/runs/rnhjdn44",
         "test_metrics": {
-            "accuracy": 0.9499,
-            "f1_score": 0.2121,
-            "precision_score": 0.2593,
-            "recall_score": 0.1795,
-            "roc_auc_score": 0.9261
+            "accuracy": 0.9589,
+            "f1_score": 0.5,
+            "precision_score": 0.5625,
+            "recall_score": 0.45,
+            "roc_auc_score": 0.9672
         },
         "val_metrics": {
-            "accuracy": 0.955,
-            "f1_score": 0.2,
-            "precision_score": 0.2727,
-            "recall_score": 0.1579,
-            "roc_auc_score": 0.9221
+            "accuracy": 0.9548,
+            "f1_score": 0.4,
+            "precision_score": 0.4,
+            "recall_score": 0.4,
+            "roc_auc_score": 0.9511
         }
     },
-    "Zambia_2019_skip_era5": {
-        "params": "https://wandb.ai/nasa-harvest/crop-mask/runs/hgvt6wqk",
+    "Zambia_2019_v2": {
+        "params": "https://wandb.ai/nasa-harvest/crop-mask/runs/gqni0fso",
         "test_metrics": {
-            "accuracy": 0.9499,
-            "f1_score": 0.2121,
-            "precision_score": 0.2593,
-            "recall_score": 0.1795,
-            "roc_auc_score": 0.9261
+            "accuracy": 0.968,
+            "f1_score": 0.65,
+            "precision_score": 0.65,
+            "recall_score": 0.65,
+            "roc_auc_score": 0.9349
         },
         "val_metrics": {
-            "accuracy": 0.955,
-            "f1_score": 0.2,
-            "precision_score": 0.2727,
-            "recall_score": 0.1579,
-            "roc_auc_score": 0.9221
+            "accuracy": 0.9724,
+            "f1_score": 0.6452,
+            "precision_score": 0.625,
+            "recall_score": 0.6667,
+            "roc_auc_score": 0.9363
         }
     }
 }
diff --git a/data/raw.dvc b/data/raw.dvc
@@ -1,6 +1,6 @@
 outs:
-- md5: f568cf8716f2b5e6f378082e2bb834e9.dir
-  size: 441701157
-  nfiles: 377
+- md5: 92c2efd6566d3511c82ef5f663b55962.dir
+  size: 439438580
+  nfiles: 370
   path: raw
   hash: md5
diff --git a/data/report.txt b/data/report.txt
@@ -220,11 +220,11 @@ eo_data_duplicate      5
 Zambia_CEO_2019 (Timesteps: 24)
 ----------------------------------------------------------------------------
 disagreement: 12.2%
-eo_data_complete          2103
-eo_data_skipped            293
-eo_data_missing_values       3
-✔ validation amount: 1066, positive class: 3.6%
-✔ testing amount: 1037, positive class: 3.8%
+eo_data_complete    2106
+eo_data_skipped      293
+✔ training amount: 1270, positive class: 3.3%
+✔ validation amount: 398, positive class: 3.8%
+✔ testing amount: 438, positive class: 4.6%
 
 
 
@@ -383,15 +383,15 @@ eo_data_complete    1500
 MaliStratifiedCEO2019 (Timesteps: 24)
 ----------------------------------------------------------------------------
 eo_data_complete    1350
-✔ training amount: 405, positive class: 4.9%
-✔ validation amount: 458, positive class: 5.9%
-✔ testing amount: 487, positive class: 2.3%
+✔ training amount: 401, positive class: 4.5%
+✔ validation amount: 481, positive class: 4.4%
+✔ testing amount: 468, positive class: 4.1%
 
 
 
 SudanAlGadarefCEO2020 (Timesteps: 24)
 ----------------------------------------------------------------------------
 eo_data_complete    1500
-✔ training amount: 449, positive class: 42.1%
-✔ validation amount: 518, positive class: 37.1%
-✔ testing amount: 533, positive class: 37.7%
+✔ training amount: 429, positive class: 37.5%
+✔ validation amount: 539, positive class: 40.6%
+✔ testing amount: 532, positive class: 38.0%
diff --git a/datasets.py b/datasets.py
@@ -908,7 +908,7 @@ def load_labels(self) -> pd.DataFrame:
                 filename="ceo-2019-Zambia-Cropland-(RCMRD-Set-1)-sample-data-2021-12-12.csv",
                 class_prob=lambda df: (df["Crop/non-crop"] == "Crop"),
                 start_year=2019,
-                train_val_test=(0.0, 0.5, 0.5),
+                train_val_test=(0.6, 0.2, 0.2),
                 latitude_col="lat",
                 longitude_col="lon",
                 filter_df=clean_ceo_data,
@@ -917,7 +917,7 @@ def load_labels(self) -> pd.DataFrame:
                 filename="ceo-2019-Zambia-Cropland-(RCMRD-Set-2)-sample-data-2021-12-12.csv",
                 class_prob=lambda df: (df["Crop/non-crop"] == "Crop"),
                 start_year=2019,
-                train_val_test=(0.0, 0.5, 0.5),
+                train_val_test=(0.6, 0.2, 0.2),
                 latitude_col="lat",
                 longitude_col="lon",
                 filter_df=clean_ceo_data,

diff --git a/src/models/model.py b/src/models/model.py
@@ -7,7 +7,7 @@
 import pandas as pd
 import pytorch_lightning as pl
 import torch
-from openmapflow.bands import ERA5_BANDS
+from openmapflow.bands import ERA5_BANDS, S1_BANDS
 from openmapflow.bbox import BBox
 from openmapflow.config import DATA_DIR, PROJECT_ROOT, DataPaths
 from openmapflow.constants import CLASS_PROB, EO_DATA, SUBSET
@@ -92,6 +92,10 @@ def __init__(self, hparams: Namespace) -> None:
 
         if "skip_era5" in hparams and hparams.skip_era5:
             self.bands_to_use = [i for i, v in enumerate(BANDS) if v not in ERA5_BANDS]
+        elif "skip_era5_s1" in hparams and hparams.skip_era5_s1:
+            self.bands_to_use = [
+                i for i, v in enumerate(BANDS) if v not in ERA5_BANDS and v not in S1_BANDS
+            ]
         else:
             self.bands_to_use = [i for i, _ in enumerate(BANDS)]
 

diff --git a/train.py b/train.py
@@ -21,7 +21,9 @@
 parser.add_argument("--seed", type=int, default=42)
 
 parser.add_argument("--skip_era5", dest="skip_era5", action="store_true")
+parser.add_argument("--skip_era5_s1", dest="skip_era5_s1", action="store_true")
 parser.set_defaults(skip_era5=False)
+parser.set_defaults(skip_era5_s1=False)
 parser.add_argument("--wandb", dest="wandb", action="store_true")
 parser.set_defaults(wandb=False)