@@ -249,17 +249,17 @@ def detect_dataset(dataset_path, task='apc', load_aug=False):
249249 dataset_file ['valid' ] += find_files (search_path , [d , 'dev' , task ], exclude_key = ['.inference' , 'train.' , 'test.' ] + filter_key_words + ['.ignore' ])
250250
251251 else :
252- print ('Try to load {} dataset from local' .format (dataset_path ))
252+ print ('Try to load {} dataset from local disk ' .format (dataset_path ))
253253 if load_aug :
254254 dataset_file ['train' ] += find_files (d , ['train' , task ], exclude_key = ['.inference' , 'test.' , 'valid.' ] + filter_key_words )
255255 dataset_file ['test' ] += find_files (d , ['test' , task ], exclude_key = ['.inference' , 'train.' , 'valid.' ] + filter_key_words )
256256 dataset_file ['valid' ] += find_files (d , ['valid' , task ], exclude_key = ['.inference' , 'train.' ] + filter_key_words )
257257 dataset_file ['valid' ] += find_files (d , ['dev' , task ], exclude_key = ['.inference' , 'train.' ] + filter_key_words )
258258 else :
259259 dataset_file ['train' ] += find_cwd_files ([d , 'train' , task ], exclude_key = ['.inference' , 'test.' , 'valid.' ] + filter_key_words + ['.ignore' ])
260- dataset_file ['test' ] += find_cwd_files ([d , 'train ' , task ], exclude_key = ['.inference' , 'train.' , 'valid.' ] + filter_key_words + ['.ignore' ])
261- dataset_file ['valid' ] += find_cwd_files ([d , 'train ' , task ], exclude_key = ['.inference' , 'train.' , 'test.' ] + filter_key_words + ['.ignore' ])
262- dataset_file ['valid' ] += find_cwd_files ([d , 'train ' , task ], exclude_key = ['.inference' , 'train.' , 'test.' ] + filter_key_words + ['.ignore' ])
260+ dataset_file ['test' ] += find_cwd_files ([d , 'test ' , task ], exclude_key = ['.inference' , 'train.' , 'valid.' ] + filter_key_words + ['.ignore' ])
261+ dataset_file ['valid' ] += find_cwd_files ([d , 'valid ' , task ], exclude_key = ['.inference' , 'train.' , 'test.' ] + filter_key_words + ['.ignore' ])
262+ dataset_file ['valid' ] += find_cwd_files ([d , 'valid ' , task ], exclude_key = ['.inference' , 'train.' , 'test.' ] + filter_key_words + ['.ignore' ])
263263
264264 # # if we need train a checkpoint using as much data as possible, we can merge train, valid and test set as training sets
265265 # dataset_file['train'] = dataset_file['train'] + dataset_file['test'] + dataset_file['valid']
0 commit comments