Commit adda4897 authored by Jiri Borovec's avatar Jiri Borovec

hot fix, unordered fn map()

parent cf7d918d
......@@ -431,7 +431,7 @@ def add_image_prob_noise(im, ration=0.1):
return im_noise
def mproc_load_images(list_path_img):
def wrapper_load_images(list_path_img):
logger.debug('parallel loading %i images', len(list_path_img))
list_names_imgs = map(load_image, list_path_img)
return list_names_imgs
......@@ -453,29 +453,30 @@ def dataset_load_images(path_dir, im_pattern='*', im_posix=IMAGE_POSIX,
assert os.path.exists(path_dir)
path_search = os.path.join(path_dir, im_pattern + im_posix)
logger.debug('image search "%s"', path_search)
path_imgs = glob.glob(path_search)
logger.debug('number spls %i in dataset "%s"', len(path_imgs),
paths_img = glob.glob(path_search)
logger.debug('number spls %i in dataset "%s"', len(paths_img),
os.path.basename(path_dir))
path_imgs = sorted(path_imgs)[:nb_spls]
paths_img = sorted(paths_img)[:nb_spls]
if nb_jobs > 1:
logger.info('running in %i threads...', nb_jobs)
nb_load_blocks = len(path_imgs) / BLOCK_NB_LOAD_IMAGES
nb_load_blocks = len(paths_img) / BLOCK_NB_LOAD_IMAGES
logger.debug('estimated %i loading blocks', nb_load_blocks)
paths_img = (path_imgs[i::nb_load_blocks] for i in range(nb_load_blocks))
block_paths_img = (paths_img[i::nb_load_blocks] for i in range(nb_load_blocks))
mproc_pool = mproc.Pool(nb_jobs)
list_names_imgs = mproc_pool.map(mproc_load_images, paths_img)
list_names_imgs = mproc_pool.map(wrapper_load_images, block_paths_img)
mproc_pool.close()
mproc_pool.join()
logger.info('transforming the parallel results')
im_names, imgs = zip(*itertools.chain(*list_names_imgs))
logger.debug('transforming the parallel results')
names_imgs = sorted(itertools.chain(*list_names_imgs))
im_names, imgs = zip(*names_imgs)
else:
logger.info('running in single thread...')
names_imgs = map(load_image, path_imgs)
logger.info('split the resulting tuples')
names_imgs = [load_image(p) for p in paths_img]
logger.debug('split the resulting tuples')
im_names, imgs = zip(*names_imgs)
assert len(path_imgs) == len(imgs)
assert len(paths_img) == len(imgs)
return imgs, im_names
......
......@@ -400,7 +400,7 @@ def alpe_pipe_atlas_learning_ptn_weights(imgs, init_atlas=None, init_weights=Non
logger.info('-> iter. #%i with Atlas diff %f', (i + 1), step_diff)
export_visual_atlas(i + 1, out_dir, atlas, out_prefix)
# stoping criterion
# stopping criterion
if step_diff <= thr_step_diff:
logger.info('>> exiting while the atlas diff %f is smaller then %f',
step_diff, thr_step_diff)
......
......@@ -190,16 +190,18 @@ class ExperimentAPD(tl_expt.Experiment):
:param gt: bool
"""
logging.info('loading required data')
self._load_images()
if gt:
self._load_data_ground_truth()
self._load_images()
assert len(self.imgs) == len(self.gt_img_rct)
logging.debug('loaded %i images', len(self.imgs))
self.imgs = [im.astype(np.uint8, copy=False) for im in self.imgs]
def _load_images(self):
""" load image data """
path_data = os.path.join(self.params.get('path_in'),
self.params.get('dataset'))
self.imgs, self._im_names = gen_data.dataset_load_images(path_data)
self.imgs = [im.astype(np.uint8) for im in self.imgs]
def run(self, gt=True, iter_var='case', iter_vals=range(1)):
""" the main_real procedure that load, perform and evaluete experiment
......@@ -263,17 +265,17 @@ class ExperimentAPD(tl_expt.Experiment):
n_csv = 'encoding{}.csv'.format(posix)
path_csv = os.path.join(self.params.get('path_exp'), n_csv)
if not hasattr(self, '_im_names'):
self._im_names = map(str, range(self.w_bins.shape[0]))
self._im_names = [str(i) for i in range(self.w_bins.shape[0])]
df = pd.DataFrame(data=self.w_bins, index=self._im_names[:len(self.w_bins)])
df.columns = ['ptn {:02d}'.format(lb + 1) for lb in df.columns]
df.index.name = 'image'
df.to_csv(path_csv)
def _compute_statistic_gt(self, img_rct=None):
def _compute_statistic_gt(self, imgs_rct=None):
""" compute the statistic gor GT and estimated atlas and reconstructed images
:param atlas: np.array<height, width>
:param img_rct: [np.array<height, width>]
:param imgs_rct: [np.array<height, width>]
:return: {str: float, }
"""
stat = {}
......@@ -284,12 +286,12 @@ class ExperimentAPD(tl_expt.Experiment):
self.atlas.ravel())
logging.debug('compute reconstruction - %s', hasattr(self, 'gt_img_rct'))
# error estimation from original reconstruction
if hasattr(self, 'gt_img_rct') and img_rct is not None:
# img_rct = ptn_dict.reconstruct_samples(self.atlas, self.w_bins)
# img_rct = self._binarize_img_reconstruction(img_rct)
diff = np.asarray(self.gt_img_rct[:len(img_rct)]) - np.asarray(img_rct)
stat['reconstruct_diff'] = \
np.sum(abs(diff)) / np.prod(np.asarray(self.gt_img_rct).shape)
if hasattr(self, 'gt_img_rct') and imgs_rct is not None:
# imgs_rct = ptn_dict.reconstruct_samples(self.atlas, self.w_bins)
# imgs_rct = self._binarize_img_reconstruction(imgs_rct)
imgs_gt = self.gt_img_rct[:len(imgs_rct)]
diff = np.asarray(imgs_gt) - np.asarray(imgs_rct)
stat['reconstruct_diff'] = np.sum(abs(diff)) / float(np.prod(diff.shape))
return stat
def _evaluate(self):
......
......@@ -2,44 +2,44 @@
# STATE-OF-THE-ART methods
# python run_experiment_apd_sta.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v0 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
#
# python run_experiment_apd_sta.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
#
# python run_experiment_apd_sta.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v2 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
#
# python run_experiment_apd_sta.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
#
# python run_experiment_apd_sta.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v0 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-3D-sta
#
# python run_experiment_apd_sta.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v1 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-3D-sta
python run_experiment_apd_sta.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v0 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
# OUR method
python run_experiment_apd_sta.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_apdl.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v0 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
python run_experiment_apd_sta.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v2 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_apdl.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
python run_experiment_apd_sta.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_apdl.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v2 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
python run_experiment_apd_sta.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v0 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-3D-sta
python run_experiment_apd_apdl.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
\ No newline at end of file
python run_experiment_apd_sta.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-3D-sta
# OUR method
#python run_experiment_apd_apdl.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v0 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
#
#python run_experiment_apd_apdl.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
#
#python run_experiment_apd_apdl.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v2 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
#
#python run_experiment_apd_apdl.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
\ No newline at end of file
......@@ -174,7 +174,7 @@ def load_segmentation(dict_params, img_name):
# # encoding from the input csv
# return compute_reconstruction(*mp_tuple)
def mproc_wrapper(mp_tuple):
def wrapper_reconstruction(mp_tuple):
# recompute encoding and then does the reconstruction
dict_params, path_out, img_atlas, weights, img_name = mp_tuple
segm_orig = load_segmentation(dict_params, img_name)
......@@ -213,11 +213,11 @@ def perform_reconstruction_mproc(dict_params, name_csv, df_encode, img_atlas):
if NB_THREADS > 1:
logger.debug('computing %i samples in %i threads', len(df_encode), NB_THREADS)
mproc_pool = mproc.Pool(NB_THREADS)
results = mproc_pool.map(mproc_wrapper, mp_tuples)
results = mproc_pool.map(wrapper_reconstruction, mp_tuples)
mproc_pool.close()
mproc_pool.join()
else:
results = map(mproc_wrapper, mp_tuples)
results = map(wrapper_reconstruction, mp_tuples)
df_diff = pd.DataFrame(results, columns=['image', name_csv])
df_diff = df_diff.set_index('image')
......
......@@ -125,7 +125,7 @@ class ExperimentLinearCombine(expt_apd.ExperimentAPD_mp):
name_posix = '_{}_{}'.format(self.iter_var_name, v)
if isinstance(self.params['nb_samples'], float):
self.params['nb_samples'] = int(len(self.imgs) * self.params['nb_samples'])
imgs_vec = np.array(map(np.ravel, self.imgs[:self.params['nb_samples']]))
imgs_vec = np.array([np.ravel(im) for im in self.imgs[:self.params['nb_samples']]])
atlas_ptns, rct_vec = self._perform_linear_combination(imgs_vec)
# img_rct = rct_vec.reshape(np.asarray(self.imgs[:self.params['nb_samples']]).shape)
self._convert_patterns_to_atlas(atlas_ptns)
......@@ -211,8 +211,8 @@ def experiments_synthetic(params=SYNTH_PARAMS):
params.update(arg_params)
params['max_iter'] = 9999
l_params = [params]
l_params = expt_apd.extend_list_params(l_params, 'nb_samples',
np.linspace(0.1, 1, 10).tolist())
# l_params = expt_apd.extend_list_params(l_params, 'nb_samples',
# np.linspace(0.1, 1, 10).tolist())
l_params = expt_apd.extend_list_params(l_params, 'dataset', SYNTH_SUB_DATASETS)
if not 'methods' in params:
......
......@@ -350,15 +350,14 @@ def segment_image(params, dict_paths, p_im):
visual_pipeline(dict_paths, img_raw, img, seg_raw, seg, n_img)
def perform_mproc(mp_set):
def wrapper_segment_image(mp_set):
if b_debug:
segment_image(*mp_set)
else:
try:
logging.debug('run segment. in try-catch mode...')
segment_image(*mp_set)
except:
pass
except: pass
def segment_image_folder(params=DEFAULT_PARAMS, dict_paths=DEFAULT_PATHS,
......@@ -381,11 +380,11 @@ def segment_image_folder(params=DEFAULT_PARAMS, dict_paths=DEFAULT_PATHS,
if nb_jobs > 1:
logger.debug('perform_sequence in %i threads', nb_jobs)
mproc_pool = mproc.Pool(nb_jobs)
mproc_pool.map(perform_mproc, mp_set)
mproc_pool.map(wrapper_segment_image, mp_set)
mproc_pool.close()
mproc_pool.join()
else:
map(perform_mproc, mp_set)
map(wrapper_segment_image, mp_set)
def mproc_visual_pair_orig_segm(mp_set):
......
......@@ -165,7 +165,7 @@ def apply_images(imgs, func, nb_jobs=NB_THREADS):
mproc_pool.join()
else:
logger.info('running single thread')
imgs_final = map(func, imgs)
imgs_final = [func(im) for im in imgs]
return imgs_final
......
"""
run experiments tests
Copyright (C) 2015-2016 Jiri Borovec <jiri.borovec@fel.cvut.cz>
"""
import logging
import run_experiment_apd_sta
import run_experiment_apd_apdl
def main():
""" main_real entry point """
logging.basicConfig(level=logging.INFO)
logging.info('running...')
run_experiment_apd_sta.experiments_test()
run_experiment_apd_apdl.experiments_test()
logging.info('DONE')
# plt.show()
if __name__ == '__main__':
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment