From c4c0a29007e4007dc38588eb0c1187844ba01af5 Mon Sep 17 00:00:00 2001 From: minus31 Date: Fri, 18 Oct 2019 15:00:05 +0900 Subject: [PATCH 1/7] Add make_generator(#33) --- keras_retinanet/utils/make_generator.py | 127 ++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 keras_retinanet/utils/make_generator.py diff --git a/keras_retinanet/utils/make_generator.py b/keras_retinanet/utils/make_generator.py new file mode 100644 index 0000000..70ae224 --- /dev/null +++ b/keras_retinanet/utils/make_generator.py @@ -0,0 +1,127 @@ +""" +Copyright 2017-2018 Fizyr (https://fizyr.com) +""" +import argparse +import os +import sys +import warnings +import keras +import keras.preprocessing.image +import tensorflow as tf + +# Allow relative imports when being executed as script. +if __name__ == "__main__" and __package__ is None: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + import keras_retinanet.bin # noqa: F401 + __package__ = "keras_retinanet.bin" + +from ..preprocessing.pascal_voc import PascalVocGenerator +from ..utils.anchors import make_shapes_callback +from ..utils.config import read_config_file, parse_anchor_parameters +from ..utils.keras_version import check_keras_version +from ..utils.transform import random_transform_generator +from ..utils.image import random_visual_effect_generator +from ..preprocessing.download import download_pascal + +def make_generators(batch_size=32, image_min_side=800, image_max_side=1333, preprocess_image=lambda x : x / 255., + random_transform=True, dataset_type='voc', vesion="2012"): + """ Create generators for training and validation. + Args/ + + args : parseargs object containing configuration for generators. + preprocess_image : Function that preprocesses an image for the network. + """ + # common_args = { + # 'batch_size' : args.batch_size, + # 'config' : args.config, + # 'image_min_side' : args.image_min_side, + # 'image_max_side' : args.image_max_side, + # 'preprocess_image' : preprocess_image, + # } + common_args = { + 'batch_size' : batch_size, + 'image_min_side' : image_min_side, + 'image_max_side' : image_max_side, + 'preprocess_image' : preprocess_image, + } + + # create random transform generator for augmenting training data + if random_transform: + transform_generator = random_transform_generator( + min_rotation=-0.1, + max_rotation=0.1, + min_translation=(-0.1, -0.1), + max_translation=(0.1, 0.1), + min_shear=-0.1, + max_shear=0.1, + min_scaling=(0.9, 0.9), + max_scaling=(1.1, 1.1), + flip_x_chance=0.5, + flip_y_chance=0.5, + ) + visual_effect_generator = random_visual_effect_generator( + contrast_range=(0.9, 1.1), + brightness_range=(-.1, .1), + hue_range=(-0.05, 0.05), + saturation_range=(0.95, 1.05) + ) + else: + transform_generator = random_transform_generator(flip_x_chance=0.5) + visual_effect_generator = None + + + # Dataset path + + dataset_path = os.path.join(os.path.expanduser("~"), ".yolk/datasets/") + + if dataset_type == 'coco': + # import here to prevent unnecessary dependency on cocoapi + + if not os.path.exists(os.path.join(dataset_path, dataset_type)): + # download_coco() + pass + + from ..preprocessing.coco import CocoGenerator + + train_generator = CocoGenerator( + os.path.join(dataset_path, dataset_type), + 'train2017', + transform_generator=transform_generator, + visual_effect_generator=visual_effect_generator, + **common_args + ) + + validation_generator = CocoGenerator( + os.path.join(dataset_path, dataset_type), + 'val2017', + shuffle_groups=False, + **common_args + ) + + elif dataset_type == 'voc': + + if not os.path.exists(os.path.join(dataset_path, dataset_type)): + download_pascal() + + train_generator = PascalVocGenerator( + os.path.join(dataset_path, dataset_type), + 'trainval', + transform_generator=transform_generator, + visual_effect_generator=visual_effect_generator, + **common_args + ) + + validation_generator = PascalVocGenerator( + os.path.join(dataset_path, dataset_type), + 'test', + shuffle_groups=False, + **common_args + ) + + else: + raise ValueError('Invalid data type received: {}'.format(dataset_type)) + + return train_generator, validation_generator + +if __name__ == "__main__": + pass \ No newline at end of file From e3559e2b793549785f47f8ba0fabb620578c49a4 Mon Sep 17 00:00:00 2001 From: minus31 Date: Fri, 18 Oct 2019 15:00:22 +0900 Subject: [PATCH 2/7] Add make_generator_test(#33) --- keras_retinanet/utils/make_generator_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 keras_retinanet/utils/make_generator_test.py diff --git a/keras_retinanet/utils/make_generator_test.py b/keras_retinanet/utils/make_generator_test.py new file mode 100644 index 0000000..d34783a --- /dev/null +++ b/keras_retinanet/utils/make_generator_test.py @@ -0,0 +1,11 @@ +import keras +import numpy as np + +from KerasObjectDetector.keras_retinanet.utils import make_generator + +train_gen, val_gen = make_generator.make_generators() + +sample = train_gen[0] + +assert sample[0].shape[0] == sample[1][0].shape[0] +print("generator created sucsessfully!") From 12733673de81ef78ffaa0a3743f3f5854f8f89d7 Mon Sep 17 00:00:00 2001 From: minus31 Date: Fri, 18 Oct 2019 15:00:57 +0900 Subject: [PATCH 3/7] Add download_pascal (#15) --- keras_retinanet/preprocessing/download.py | 157 ++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 keras_retinanet/preprocessing/download.py diff --git a/keras_retinanet/preprocessing/download.py b/keras_retinanet/preprocessing/download.py new file mode 100644 index 0000000..94b8c51 --- /dev/null +++ b/keras_retinanet/preprocessing/download.py @@ -0,0 +1,157 @@ +import os +import errno +import hashlib +import requests +from tqdm import tqdm + + +def makedirs(path): + """Create directory recursively if not exists. + Similar to `makedir -p`, you can skip checking existence before this function. + + Parameters + ---------- + path : str + Path of the desired dir + """ + + try: + os.makedirs(path) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + +"""Download files with progress bar.""" + +def check_sha1(filename, sha1_hash): + """Check whether the sha1 hash of the file content matches the expected hash. + Parameters + ---------- + filename : str + Path to the file. + sha1_hash : str + Expected sha1 hash in hexadecimal digits. + Returns + ------- + bool + Whether the file content matches the expected hash. + """ + sha1 = hashlib.sha1() + with open(filename, 'rb') as f: + while True: + data = f.read(1048576) + if not data: + break + sha1.update(data) + + sha1_file = sha1.hexdigest() + l = min(len(sha1_file), len(sha1_hash)) + return sha1.hexdigest()[0:l] == sha1_hash[0:l] + +def download(url, path=None, overwrite=False, sha1_hash=None): + """Download an given URL + Parameters + ---------- + url : str + URL to download + path : str, optional + Destination path to store downloaded file. By default stores to the + current directory with same name as in url. + overwrite : bool, optional + Whether to overwrite destination file if already exists. + sha1_hash : str, optional + Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified + but doesn't match. + Returns + ------- + str + The file path of the downloaded file. + """ + if path is None: + fname = url.split('/')[-1] + else: + path = os.path.expanduser(path) + if os.path.isdir(path): + fname = os.path.join(path, url.split('/')[-1]) + else: + fname = path + + if overwrite or not os.path.exists(fname) or (sha1_hash and not check_sha1(fname, sha1_hash)): + dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname))) + if not os.path.exists(dirname): + os.makedirs(dirname) + + print('Downloading %s from %s...'%(fname, url)) + r = requests.get(url, stream=True) + if r.status_code != 200: + raise RuntimeError("Failed downloading url %s"%url) + total_length = r.headers.get('content-length') + with open(fname, 'wb') as f: + if total_length is None: # no content length header + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + else: + total_length = int(total_length) + for chunk in tqdm(r.iter_content(chunk_size=1024), + total=int(total_length / 1024. + 0.5), + unit='KB', unit_scale=False, dynamic_ncols=True): + f.write(chunk) + + if sha1_hash and not check_sha1(fname, sha1_hash): + raise UserWarning('File {} is downloaded but the content hash does not match. ' \ + 'The repo may be outdated or download may be incomplete. ' \ + 'If the "repo_url" is overridden, consider switching to ' \ + 'the default repo.'.format(fname)) + + return fname + +"""Prepare PASCAL VOC datasets""" +import os +import shutil +import argparse +import tarfile + +_TARGET_DIR = os.path.expanduser('~/.yolk/datasets/voc') + +##################################################################################### +# Download and extract VOC datasets into ``path`` + +def download_voc(path, overwrite=False): + _DOWNLOAD_URLS = [ + ('http://bit.ly/yolk_voc_train_val2007_tar'), + ('http://bit.ly/yolk_voc_train_val2012_tar'), + ('http://bit.ly/yolk_voc_test2012_tar')] + makedirs(path) + for url, checksum in _DOWNLOAD_URLS: + filename = download(url, path=path, overwrite=overwrite, sha1_hash=checksum) + # extract + with tarfile.open(filename) as tar: + tar.extractall(path=path) + + +##################################################################################### +# Download and extract the VOC augmented segmentation dataset into ``path`` + +def download_pascal(download_dir="~/VOCdevkit", overwrite=True, no_download=False): + path = os.path.expanduser(download_dir) + if not os.path.isdir(path) or not os.path.isdir(os.path.join(path, 'VOC2007')) \ + or not os.path.isdir(os.path.join(path, 'VOC2012')): + if no_download: + raise ValueError(('{} is not a valid directory, make sure it is present.' + ' Or you should not disable "--no-download" to grab it'.format(path))) + else: + download_voc(path, overwrite=overwrite) + shutil.move(os.path.join(path, 'VOCdevkit', 'VOC2007'), os.path.join(path, 'VOC2007')) + shutil.move(os.path.join(path, 'VOCdevkit', 'VOC2012'), os.path.join(path, 'VOC2012')) + shutil.rmtree(os.path.join(path, 'VOCdevkit')) + + # make symlink + makedirs(os.path.expanduser('~/.yolk/datasets')) + if os.path.isdir(_TARGET_DIR): + os.remove(_TARGET_DIR) + os.symlink(path, _TARGET_DIR) + print("Downloaded!!!") + +if __name__ == "__main__": + pass \ No newline at end of file From 119496e352b277689a9705757547f081ab0fcae2 Mon Sep 17 00:00:00 2001 From: minus31 Date: Sun, 20 Oct 2019 15:38:37 +0900 Subject: [PATCH 4/7] Delete make_generator_test.py - This function moved into make_generator.py --- keras_retinanet/utils/make_generator_test.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 keras_retinanet/utils/make_generator_test.py diff --git a/keras_retinanet/utils/make_generator_test.py b/keras_retinanet/utils/make_generator_test.py deleted file mode 100644 index d34783a..0000000 --- a/keras_retinanet/utils/make_generator_test.py +++ /dev/null @@ -1,11 +0,0 @@ -import keras -import numpy as np - -from KerasObjectDetector.keras_retinanet.utils import make_generator - -train_gen, val_gen = make_generator.make_generators() - -sample = train_gen[0] - -assert sample[0].shape[0] == sample[1][0].shape[0] -print("generator created sucsessfully!") From 4c3971c847b871762bb7801cf5d579300b7d8d45 Mon Sep 17 00:00:00 2001 From: minus31 Date: Sun, 20 Oct 2019 15:39:30 +0900 Subject: [PATCH 5/7] Add make_generator_test function --- keras_retinanet/utils/make_generator.py | 28 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/keras_retinanet/utils/make_generator.py b/keras_retinanet/utils/make_generator.py index 70ae224..612c2fd 100644 --- a/keras_retinanet/utils/make_generator.py +++ b/keras_retinanet/utils/make_generator.py @@ -31,13 +31,7 @@ def make_generators(batch_size=32, image_min_side=800, image_max_side=1333, prep args : parseargs object containing configuration for generators. preprocess_image : Function that preprocesses an image for the network. """ - # common_args = { - # 'batch_size' : args.batch_size, - # 'config' : args.config, - # 'image_min_side' : args.image_min_side, - # 'image_max_side' : args.image_max_side, - # 'preprocess_image' : preprocess_image, - # } + common_args = { 'batch_size' : batch_size, 'image_min_side' : image_min_side, @@ -123,5 +117,23 @@ def make_generators(batch_size=32, image_min_side=800, image_max_side=1333, prep return train_generator, validation_generator + +def make_generator_test(): + train_gen, val_gen = make_generators() + + sample = train_gen[0] + + assert sample[0].shape[0] == sample[1][0].shape[0] + print("generator created sucsessfully!") + +import argparse + if __name__ == "__main__": - pass \ No newline at end of file + parser = argparse.ArgumentParser() + parser.add_argument('--test', type=bool, default=False, help='dataset directory on disk') + args = parser.parse_args() + + if args.test: + make_generator_test() + + \ No newline at end of file From a302213cf5a7591c181bccdba03d391103a96a35 Mon Sep 17 00:00:00 2001 From: minus31 Date: Sun, 20 Oct 2019 15:41:53 +0900 Subject: [PATCH 6/7] Add usage description --- keras_retinanet/utils/make_generator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/keras_retinanet/utils/make_generator.py b/keras_retinanet/utils/make_generator.py index 612c2fd..318d97e 100644 --- a/keras_retinanet/utils/make_generator.py +++ b/keras_retinanet/utils/make_generator.py @@ -119,6 +119,11 @@ def make_generators(batch_size=32, image_min_side=800, image_max_side=1333, prep def make_generator_test(): + """ + Testing make_generator fucntion + + usage : python make_generator.py --test True + """ train_gen, val_gen = make_generators() sample = train_gen[0] From 3a8d7ef83e6b48a161ea064b72ac4be630a9eac0 Mon Sep 17 00:00:00 2001 From: minus31 Date: Sun, 20 Oct 2019 15:57:17 +0900 Subject: [PATCH 7/7] Add shua code --- keras_retinanet/preprocessing/download.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/keras_retinanet/preprocessing/download.py b/keras_retinanet/preprocessing/download.py index 94b8c51..116fdee 100644 --- a/keras_retinanet/preprocessing/download.py +++ b/keras_retinanet/preprocessing/download.py @@ -119,9 +119,12 @@ def download(url, path=None, overwrite=False, sha1_hash=None): def download_voc(path, overwrite=False): _DOWNLOAD_URLS = [ - ('http://bit.ly/yolk_voc_train_val2007_tar'), - ('http://bit.ly/yolk_voc_train_val2012_tar'), - ('http://bit.ly/yolk_voc_test2012_tar')] + ('http://bit.ly/yolk_voc_train_val2007_tar', + '34ed68851bce2a36e2a223fa52c661d592c66b3c'), + ('http://bit.ly/yolk_voc_train_val2012_tar', + '41a8d6e12baa5ab18ee7f8f8029b9e11805b4ef1'), + ('http://bit.ly/yolk_voc_test2012_tar', + '4e443f8a2eca6b1dac8a6c57641b67dd40621a49')] makedirs(path) for url, checksum in _DOWNLOAD_URLS: filename = download(url, path=path, overwrite=overwrite, sha1_hash=checksum)