adding competitor code

Author: nrweir

.gitignore

@@ -0,0 +1 @@
+.DS_Store

MaksimovKA_solution/.dockerignore

@@ -0,0 +1 @@
+./Dockerfile

MaksimovKA_solution/Dockerfile

@@ -0,0 +1,20 @@
+# use with GPU and nvidia-docker2
+FROM tensorflow/tensorflow:1.9.0-devel-gpu-py3
+RUN add-apt-repository -y ppa:ubuntugis/ppa; \
+apt-get update && apt-get -y install mc python-opencv tmux nano python3-tk gdal-bin python3-gdal less; \
+pip3 install --upgrade pip
+RUN pip3 install h5py scikit-image scipy pillow
+RUN pip3 install opencv-python matplotlib tqdm keras==2.1.6
+RUN pip3 install keras-resnet six scipy gdal
+RUN pip install albumentations
+RUN pip install keras-resnet scikit-learn pandas
+RUN pip install jupyter
+RUN pip install gdal rasterio
+RUN pip install shapely
+RUN mkdir /project
+COPY ./ /project/
+RUN chmod +x /project/train.sh
+RUN chmod +x /project/test.sh
+WORKDIR /project
+
+ENV PYTHONPATH "${PYTHONPATH}:/project"

MaksimovKA_solution/augmentations/transforms.py

@@ -0,0 +1,26 @@
+from albumentations import (
+                            Compose,
+                            HorizontalFlip,
+                            ShiftScaleRotate,
+                            OneOf,
+                            RandomContrast,
+                            RandomGamma,
+                            RandomBrightness
+   
+                            )
+
+
+def augmentations(prob=0.5):
+    
+    transformer = Compose([
+                      
+            HorizontalFlip(p=prob),
+            ShiftScaleRotate(p=prob, shift_limit=0.1, scale_limit=.1, rotate_limit=10),
+            OneOf([RandomContrast(limit=0.1, p=prob),
+                   RandomGamma(gamma_limit=(90, 110), p=prob),
+                   RandomBrightness(limit=0.1, p=prob)],p=prob),
+            
+    ], p=prob)
+    return transformer
+
+

MaksimovKA_solution/dataset/base_dataset.py

@@ -0,0 +1,97 @@
+import os
+import numpy as np
+from keras.preprocessing.image import Iterator
+import skimage.io
+
+
+class BaseMaskDatasetIterator(Iterator):
+    def __init__(self,
+                 images_dir,
+                 masks_dir,
+                 image_ids,
+                 crop_shape,
+                 preprocessing_function,
+                 random_transformer=None,
+                 batch_size=8,
+                 shuffle=True,
+                 image_name_template=None,
+                 mask_name_template=None,
+                 add_contours=True
+                ):
+        self.images_dir = images_dir
+        self.masks_dir = masks_dir
+        self.image_ids = image_ids
+        self.image_name_template = image_name_template
+        self.mask_name_template = mask_name_template
+        self.random_transformer = random_transformer
+        self.crop_shape = crop_shape
+        self.preprocessing_function = preprocessing_function
+        self.add_contours = add_contours
+        super(BaseMaskDatasetIterator, self).__init__(len(self.image_ids), batch_size, shuffle, None)
+    
+    def pad_mask_image(self, mask, image, img_id, crop_shape):
+        # _mask = mask[:, :, 0]
+        return NotImplementedError
+
+    def transform_batch_y(self, batch_y):
+        return batch_y
+
+    def _get_batches_of_transformed_samples(self, index_array):
+        batch_x = []
+        batch_y = []
+
+        for batch_index, image_index in enumerate(index_array):
+            _id = self.image_ids[image_index]
+            # print(_id)
+            img_name = self.image_name_template.format(id=_id)
+            mask_name = self.mask_name_template.format(id=_id)
+
+            image = skimage.io.imread(os.path.join(self.images_dir, img_name), plugin='tifffile')
+            mask = skimage.io.imread(os.path.join(self.masks_dir, mask_name), plugin='tifffile')
+            if self.add_contours:
+                tmp = np.zeros(mask.shape[:2], dtype=mask.dtype)
+                mask = np.dstack([mask, tmp])
+            else:
+                mask = np.dstack([mask]*3)
+
+            crop_mask, crop_image = self.pad_mask_image(mask, image, _id, self.crop_shape)
+            if self.random_transformer is not None:
+                data = self.random_transformer(image=crop_image, mask=crop_mask)
+                crop_image, crop_mask = data['image'], data['mask']
+
+            if self.add_contours:
+                crop_mask = crop_mask / 255.
+                crop_mask = crop_mask[:, :, :2]
+            else:
+                crop_mask = crop_mask / 255.
+                crop_mask = crop_mask[:, :, 0]
+                crop_mask = np.expand_dims(crop_mask, -1)
+
+            batch_x.append(crop_image)
+            batch_y.append(crop_mask)
+            
+        batch_x = np.array(batch_x, dtype="float32")
+        batch_y = np.array(batch_y, dtype="float32")
+        if self.preprocessing_function:
+            batch_x = self.preprocess_input(batch_x)
+
+        return self.transform_batch_x(batch_x), self.transform_batch_y(batch_y)
+
+    @staticmethod
+    def preprocess_input(batch_x):
+        means = [963.0, 805.0, 666.0]
+        stds = [473.0, 403.0, 395.0]
+        for i in range(batch_x.shape[3]):
+            batch_x[:, :, :, i] = (batch_x[:, :, :, i] - means[i]) / stds[i]
+        return batch_x
+
+    def transform_batch_x(self, batch_x):
+        return batch_x
+
+    def next(self):
+
+        with self.lock:
+            index_array = next(self.index_generator)
+        return self._get_batches_of_transformed_samples(index_array)
+
+

MaksimovKA_solution/dataset/spacenet_binary_dataset.py

@@ -0,0 +1,112 @@
+import numpy as np
+import pandas as pd
+from .base_dataset import BaseMaskDatasetIterator
+from albumentations import RandomCrop, Compose, PadIfNeeded
+
+class SpacenetBinaryDataset:
+    def __init__(self,
+                 images_dir,
+                 masks_dir,
+                 folds_file,
+                 fold=0,
+                 fold_num=5,
+                 add_contours=False
+                 ):
+        super().__init__()
+        self.fold = fold
+        self.folds_file = folds_file
+        self.fold_num = fold_num
+        self.images_dir = images_dir
+        self.masks_dir = masks_dir
+        self.add_contours = add_contours
+        self.train_ids, self.val_ids = self.generate_ids()
+        print("Found {} train images".format(len(self.train_ids)))
+        print("Found {} val images".format(len(self.val_ids)))
+
+    def get_generator(self, image_ids, crop_shape, preprocessing_function=1,
+                      random_transformer=None, batch_size=16, shuffle=True):
+        return SpacenetDatasetIterator(
+            self.images_dir,
+            self.masks_dir,
+            image_ids,
+            crop_shape,
+            preprocessing_function,
+            random_transformer,
+            batch_size,
+            shuffle=shuffle,
+            image_name_template="{id}.tif",
+            mask_name_template="{id}.tif",
+            add_contours=self.add_contours
+        )
+
+    def train_generator(self, crop_shape, preprocessing_function=1, random_transformer=None, batch_size=16):
+        return self.get_generator(self.train_ids, crop_shape, preprocessing_function,
+                                  random_transformer, batch_size, True)
+
+    def val_generator(self, preprocessing_function=1, batch_size=1):
+        return self.get_generator(self.val_ids, (928, 928), preprocessing_function, None, batch_size, False)
+
+    def generate_ids(self):
+        
+        df = pd.read_csv(self.folds_file)
+        
+        df['angle'] = df['img_id'].apply(lambda x: int(x.split('_')[1][5:]))
+        # df = df[(df['angle'] > 25) & (df['angle'] <= 40)]
+        # df = df[df['angle'] > 40]
+        val_ids = df[(df['fold_on_train'] == self.fold)]['img_id'].values
+        train_ids = np.sort(df[(df['fold_on_predict'] != self.fold)]['img_id'].values)
+        return train_ids, val_ids
+
+
+class SpacenetDatasetIterator(BaseMaskDatasetIterator):
+
+    def __init__(self,
+                 images_dir,
+                 masks_dir,
+                 image_ids,
+                 crop_shape,
+                 preprocessing_function,
+                 random_transformer=None,
+                 batch_size=8,
+                 shuffle=True,
+                 image_name_template=None,
+                 mask_name_template=None,
+                 add_contours=False):
+        super().__init__(images_dir,
+                         masks_dir,
+                         image_ids,
+                         crop_shape,
+                         preprocessing_function,
+                         random_transformer,
+                         batch_size,
+                         shuffle,
+                         image_name_template,
+                         mask_name_template,
+                         add_contours)
+
+    def pad_mask_image(self, mask, image, img_id, crop_shape):
+        composed = Compose([PadIfNeeded(crop_shape[0], crop_shape[1], p=1),
+                            RandomCrop(crop_shape[0], crop_shape[1], p=1)], p=1)
+
+        if np.sum(mask) != 0:
+
+            s = 0
+            tries = 0
+            while s == 0:
+                # crop = composed(crop_shape[0], crop_shape[1])
+                croped = composed(image=image, mask=mask)
+
+                image_padded = croped['image']
+                mask_padded = croped['mask']
+                # print(mask_padded.shape)
+                s = np.sum(mask_padded)
+                tries += 1
+                if tries > 5:
+                    break
+        else:
+
+            croped = composed(image=image, mask=mask)
+            image_padded = croped['image']
+            mask_padded = croped['mask']
+            
+        return mask_padded, image_padded