Skip to content

Commit 7f79d3a

Browse files
author
Bobby Jackson
committed
ADD: Newest processing scripts
1 parent d2aa5bc commit 7f79d3a

14 files changed

+283
-144
lines changed

‎scripts/preprocess_radar.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@
1212

1313

1414
radar_grids_path = '/home/rjackson/bebop_earthscience/rjackson/deep_learning/2006/'
15-
tfrecords_path = '/home/rjackson/tfrecords/2006/2frames_future/'
15+
tfrecords_path = '/home/rjackson/tfrecords/2006/3frames_future/'
1616
num_frames_in_past = 3
1717
my_shape = (201, 201)
1818
is_training = True
1919
shuffle = False
2020

21-
def create_tf_record(radar_list, scan_no, num_previous_frames=3, num_frames_future=2):
21+
def create_tf_record(radar_list, scan_no, num_previous_frames=3, num_frames_future=3):
2222
# Get radar file from bebop
2323
if scan_no < num_previous_frames:
2424
print("Not enough frames in the past.")

‎scripts/preprocess_radar_new.py

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import tensorflow as tf
2+
from tensorflow.keras.models import Sequential, load_model, Model
3+
from tensorflow.keras.layers import Dense, Activation, Embedding, Flatten, Dropout, TimeDistributed, Reshape, Lambda
4+
from tensorflow.keras.layers import LSTM, ConvLSTM2D, Conv2D, ZeroPadding2D
5+
import numpy as np
6+
import xarray as xr
7+
from glob import glob
8+
import matplotlib.pyplot as plt
9+
import warnings
10+
from datetime import datetime, timedelta
11+
warnings.filterwarnings("ignore")
12+
13+
14+
radar_grids_path = '/home/rjackson/bebop_earthscience/rjackson/deep_learning/2005/'
15+
tfrecords_path = '/home/rjackson/tfrecords/2005/'
16+
num_frames_in_past = 3
17+
my_shape = (201, 201)
18+
is_training = True
19+
shuffle = False
20+
21+
def create_tf_record(radar_list, scan_no):
22+
# Get radar file from bebop
23+
24+
25+
# First get previous frames
26+
# Normalization: -20 dBZ = 0, 60 dBZ = 1
27+
try:
28+
grid = xr.open_dataset(radar_list[scan_no])
29+
Zn = grid.Znorm.fillna(-20).values
30+
if (np.nanmax(Zn) > 0):
31+
Zn = (Zn + 20.)/(80)
32+
else:
33+
Zn = np.zeros_like(Zn)
34+
except:
35+
return
36+
times = grid.time.values
37+
grid.close()
38+
shp = Zn.shape
39+
my_shape = shp
40+
width = shp[0]
41+
height = shp[1]
42+
43+
#if dt_future > timedelta(minutes=60):
44+
# print("Data not continous")
45+
# return
46+
47+
fname = tfrecords_path + radar_list[scan_no][-16:] + '.tfrecord'
48+
writer = tf.python_io.TFRecordWriter(fname)
49+
50+
#norm = norm.SerializeToString()
51+
example = tf.train.Example(features=tf.train.Features(
52+
feature={
53+
'width': _int64_feature(width),
54+
'height': _int64_feature(height),
55+
'image_raw': _bytes_feature(Zn),
56+
'time': _float_feature(times),
57+
}))
58+
writer.write(example.SerializeToString())
59+
print(times)
60+
del writer
61+
62+
def _int64_feature(value):
63+
"""Creates a tf.Train.Feature from an int64 value."""
64+
if value is None:
65+
value = []
66+
if not isinstance(value, list):
67+
value = [value]
68+
return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
69+
70+
71+
def _bytes_feature(value):
72+
"""Creates a tf.Train.Feature from a bytes value."""
73+
if value is None:
74+
value = []
75+
if isinstance(value, np.ndarray):
76+
value = value.reshape(-1)
77+
value = bytes(value)
78+
if not isinstance(value, list):
79+
value = [value]
80+
return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
81+
82+
def _float_feature(value):
83+
"""Creates a tf.Train.Feature from a bytes value."""
84+
if value is None:
85+
value = []
86+
if isinstance(value, np.ndarray):
87+
value = value.reshape(-1)
88+
value = bytes(value)
89+
if not isinstance(value, list):
90+
value = [value]
91+
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
92+
93+
if __name__ == "__main__":
94+
file_list = sorted(glob(radar_grids_path + '/**/*.cdf', recursive=True))
95+
print("About to process %d files" % len(file_list))
96+
for i in range(len(file_list)):
97+
create_tf_record(file_list, i)

‎scripts/qsub_nbserver.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
#COBALT --time=5:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
cd ~
8+
source activate pyart_ml
9+
jupyter notebook --port 8080

‎scripts/qsub_train.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_one_node.py 5

‎scripts/qsub_train_1_frame.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_old_epoch.py 100 1

‎scripts/qsub_train_2_frame.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_old_epoch.py 100 2

‎scripts/qsub_train_3_frame.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_old_epoch.py 100 3

‎scripts/qsub_train_4_frame.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_old_epoch.py 100 4

‎scripts/qsub_train_5_frame.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_old_epoch.py 153 5

‎scripts/qsub_train_6_frame.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
#COBALT --time=12:00:00
3+
#COBALT -n 1
4+
echo "Starting Jupyter notebook server"
5+
NODES=`cat $COBALT_NODEFILE | wc -l`
6+
PROCS=$((NODES * 12))
7+
source activate pyart_ml
8+
python train_radar_old_epoch.py 100 6

‎scripts/train_radar.py

+13-20
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
tfrecords_path = '/home/rjackson/tfrecords/2006/*'
1818
num_frames_in_past = 3
19+
num_frames_in_future = 1
1920
my_shape = (201, 201)
2021
is_training = True
2122
shuffle = False
@@ -25,37 +26,28 @@ def parse_record(record):
2526
feature={'width': tf.FixedLenFeature([], tf.int64, default_value=0),
2627
'height': tf.FixedLenFeature([], tf.int64, default_value=0),
2728
'image_raw': tf.FixedLenFeature([], tf.string, default_value=""),
28-
'label': tf.FixedLenFeature([], tf.string, default_value=""),
2929
'num_frames_in_past': tf.FixedLenFeature([], tf.int64, default_value=0),
3030
'num_frames_in_future': tf.FixedLenFeature([], tf.int64, default_value=0),
3131
'dt_past': tf.FixedLenFeature([], tf.string, default_value=""),
32+
'time': tf.FixedLenFeature([], tf.string, default_value=""),
3233
'dt_future': tf.FixedLenFeature([], tf.float32, default_value=0.),
3334
}
3435
features = tf.io.parse_single_example(record, feature)
3536
image_shape = (num_frames_in_past, my_shape[0], my_shape[1], 1)
3637
features['image_raw'] = tf.decode_raw(features['image_raw'], tf.float32)
3738
features['image_raw'] = tf.reshape(features['image_raw'], shape=list(image_shape))
38-
features['label'] = tf.decode_raw(features['label'], tf.float32)
39-
features['label'] = tf.reshape(features['label'], shape=[image_shape[1], image_shape[2], 1])
40-
return {'conv_lst_m2d_input':features['image_raw']}, {'conv2d':features['label']}
39+
40+
return {'image_raw': features['image_raw']}
4141

42-
filelist = tf.data.Dataset.list_files(
43-
tfrecords_path,
44-
seed=3,
45-
shuffle=False)
42+
def
43+
dataset = tf.data.TFRecordDataset(file_list)
4644

47-
dataset = filelist.apply(
48-
tf.data.experimental.parallel_interleave(
49-
tf.data.TFRecordDataset,
50-
cycle_length=4,
51-
sloppy=True))
52-
53-
#if is_training:
54-
# if shuffle:
55-
# dataset = dataset.apply(tf.data.experimental.shuffle_and_repeat(
56-
# buffer_size=shuffle_buffer, seed=seed))
57-
# else:
58-
# dataset = dataset.repeat()
45+
if is_training:
46+
if shuffle:
47+
dataset = dataset.apply(tf.data.experimental.shuffle_and_repeat(
48+
buffer_size=shuffle_buffer, seed=seed))
49+
else:
50+
dataset = dataset.repeat()
5951

6052
dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
6153
dataset = dataset.apply(
@@ -64,6 +56,7 @@ def parse_record(record):
6456
batch_size=30,
6557
num_parallel_batches=4,
6658
drop_remainder=True))
59+
6760
return dataset
6861

6962
def _int64_feature(value):

‎scripts/train_radar_2frames.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
warnings.filterwarnings("ignore")
1414

1515

16-
tfrecords_path = '/home/rjackson/tfrecords/2006/*'
16+
tfrecords_path = '/home/rjackson/tfrecords/2006/2frames_future/*'
1717
num_frames_in_past = 4
1818
my_shape = (201, 201)
1919
is_training = True

0 commit comments

Comments
 (0)