Commit 4cc5fc9a authored by Jirka's avatar Jirka

experiments in parallel

parent 0a96320a
......@@ -7,8 +7,8 @@ import matplotlib.pyplot as plt
import logging
logger = logging.getLogger(__name__)
DEFAULT_UNARY_BACKGROUND = 1
DEFAULT_UNARY_BACKGROUND = 1
# TODO: spatial clustering
......@@ -317,14 +317,16 @@ def alpe_repaire_atlas_weights(imgs, atlas, w_bins, lb_max):
return atlas, w_bins
def alpe_update_atlas(imgs, atlas, w_bins, lb_max, gc_coef, gc_reinit):
def alpe_update_atlas(imgs, atlas, w_bins, lb_max, gc_coef, gc_reinit, ptn_split):
""" single iteration of the block coordinate descent algo
:param imgs: [np.array<w, h>]
:param atlas: np.array<w, h>
:param w_bins: np.array<nb_imgs, nb_lbs>
:param lb_max: int
:param gc_coef: float, graph cut regularisation
:param gc_reinit: bool, weather use atlas from previous step as init for act.
:param ptn_split: bool
:return: np.array<w, h>, float
"""
if np.sum(w_bins) == 0:
......@@ -338,16 +340,17 @@ def alpe_update_atlas(imgs, atlas, w_bins, lb_max, gc_coef, gc_reinit):
else:
atlas_new = estimate_atlas_graphcut_general(imgs, w_bins, gc_coef)
atlas_new = ptn_dict.atlas_split_indep_ptn(atlas_new, lb_max)
if ptn_split:
atlas_new = ptn_dict.atlas_split_indep_ptn(atlas_new, lb_max)
step_diff = sim_metric.compare_atlas_adjusted_rand(atlas, atlas_new)
return atlas_new, step_diff
def alpe_pipe_atlas_learning_ptn_weights(imgs, init_atlas=None, init_weights=None,
gc_coef=0.0, thr_step_diff=0.0, max_iter=99,
gc_reinit=True, out_prefix='debug', out_dir=''):
""" the main pipeline for block coordinate descent algo with graphcut
gc_coef=0.0, thr_step_diff=0.0, max_iter=99, gc_reinit=True,
ptn_split=True, out_prefix='debug', out_dir=''):
""" the experiments_all pipeline for block coordinate descent algo with graphcut
:param imgs: [np.array<w, h>]
:param init_atlas: np.array<w, h>
......@@ -381,7 +384,8 @@ def alpe_pipe_atlas_learning_ptn_weights(imgs, init_atlas=None, init_weights=Non
# plt.subplot(223), plt.imshow(atlas, interpolation='nearest')
# plt.subplot(224), plt.imshow(w_bins, aspect='auto')
# plt.show()
atlas, step_diff = alpe_update_atlas(imgs, atlas, w_bins, lb_max, gc_coef, gc_reinit)
atlas, step_diff = alpe_update_atlas(imgs, atlas, w_bins, lb_max,
gc_coef, gc_reinit, ptn_split)
logger.info('-> iter. #{} with Atlas diff {}'.format(i + 1, step_diff))
export_visual_atlas(i + 1, out_dir, atlas, w_bins, prefix=out_prefix)
......
# to suppres all visu, has to be on the beginning
import matplotlib
matplotlib.use('Agg')
import os
import sys
import time
import traceback
import numpy as np
import pandas as pd
import matplotlib.pylab as plt
import matplotlib.gridspec as gridspec
import multiprocessing as mproc
from sklearn import metrics
# import src.ownUtils.toolDataIO as tD
import generate_dataset as gen_data
......@@ -13,29 +18,39 @@ import ptn_disctionary as ptn_dict
import ptn_weights as ptn_weigth
import src.ownUtils.toolDataIO as tl_data
import src.ownUtils.toolExperiment as tl_expt
import logging
logger = logging.getLogger(__name__)
jirka = False
if jirka:
DEFAULT_PATH_DATA = '/home/jirka/TEMP/APD_synthetic_data'
DEFAULT_PATH_OUTPUT = '/home/jirka/TEMP/experiments_APD'
else:
DEFAULT_PATH_DATA = '/datagrid/Medical/microscopy/drosophila/synthetic_data'
DEFAULT_PATH_OUTPUT = '/datagrid/temporary/Medical/experiments_APD'
# DEFAULT_PATH_OUTPUT = os.path.join('..','..','output')
DEFAULT_PATH_DATA = '/datagrid/Medical/microscopy/drosophila/synthetic_data'
DEFAULT_PATH_OUTPUT = '/datagrid/temporary/Medical/experiments_APD'
DEFAULT_DATASET_NAME = 'atomicPatternDictionary_v0'
DEFAULT_PATH_APD = os.path.join(DEFAULT_PATH_DATA, DEFAULT_DATASET_NAME)
DEFAULT_SUB_DATASETS = ['datasetBinary_raw',
'datasetBinary_noise',
'datasetBinary_deform',
'datasetBinary_defNoise',]
DEFAULT_PARAMS = {
'name': 'NAME',
'in_path': DEFAULT_PATH_DATA,
'dataset': DEFAULT_DATASET_NAME,
'sub_dataset': gen_data.DEFAULT_DATASET,
'sub_dataset': DEFAULT_SUB_DATASETS[0],
'out_path': DEFAULT_PATH_OUTPUT,
'init_tp': 'rnd', # msc. rnd
'max_iter': 99,
'init_tp': 'msc', # msc. rnd
'max_iter': 29,
'gc_regul': 0.,
'nb_lbs': 7,
'nb_runs': 25, # 500
'nb_runs': 500, # 500
'gc_reinit': True,
'ptn_split': True,
}
def test_simple_show_case():
# implement simple case just with 2 images and 2/3 classes in atlas
atlas = gen_data.get_simple_atlas()
......@@ -71,11 +86,12 @@ def test_simple_show_case():
def experiment_pipeline_alpe_showcase(p_out=DEFAULT_PATH_OUTPUT):
atlas = gen_data.dataset_create_atlas()
atlas = gen_data.dataset_create_atlas(path_base=DEFAULT_PATH_APD)
# plt.imshow(atlas)
imgs = gen_data.dataset_load_images()
# imgs = gen_data.dataset_load_images('datasetBinary_defNoise')
imgs = gen_data.dataset_load_images(path_base=DEFAULT_PATH_APD)
# imgs = gen_data.dataset_load_images('datasetBinary_defNoise',
# path_base=DEFAULT_PATH_APD)
init_atlas_org = ptn_dict.initialise_atlas_deform_original(atlas)
init_atlas_rnd = ptn_dict.initialise_atlas_random(atlas.shape, np.max(atlas))
......@@ -88,6 +104,13 @@ def experiment_pipeline_alpe_showcase(p_out=DEFAULT_PATH_OUTPUT):
def alpe_init_atlas(im_size, nb_lbs, init_tp):
"""
:param im_size:
:param nb_lbs:
:param init_tp:
:return:
"""
if init_tp == 'msc':
init_atlas = ptn_dict.initialise_atlas_mosaic(im_size, nb_lbs)
else:
......@@ -96,24 +119,47 @@ def alpe_init_atlas(im_size, nb_lbs, init_tp):
def alpe_compute_stat(GT_atlas, GT_img_rct, atlas, w_bins):
"""
:param GT_atlas:
:param GT_img_rct:
:param atlas:
:param w_bins:
:return:
"""
ars = metrics.adjusted_rand_score(GT_atlas.ravel(), atlas.ravel())
img_rct = ptn_dict.reconstruct_samples(atlas, w_bins)
diff_rct = np.sum(abs(np.asarray(GT_img_rct) - np.asarray(img_rct)))
stat = {'atlas_ARS': ars, 'recosntruct_diff': diff_rct}
diff_rct /= np.prod(np.asarray(GT_img_rct).shape)
stat = {'atlas_ARS': ars, 'reconstruct_diff': diff_rct}
return stat
def alpe_pipe_atlas_learning_ptn_weights(i, imgs, init_atlas, params):
def alpe_pipe_atlas(i, imgs, init_atlas, params):
prefix = 'expt_{}'.format(params['init_tp'])
p_out = os.path.join(params['res_path'], 'case_{:05d}'.format(i))
if not os.path.exists(p_out):
os.mkdir(p_out)
return dl.alpe_pipe_atlas_learning_ptn_weights(imgs, init_atlas=init_atlas,
gc_reinit=params['gc_reinit'], gc_coef=params['gc_regul'],
max_iter=params['max_iter'], out_dir=p_out, out_prefix=prefix)
try:
atlas, w_bins = \
dl.alpe_pipe_atlas_learning_ptn_weights(imgs, init_atlas=init_atlas,
gc_reinit=params['gc_reinit'], gc_coef=params['gc_regul'],
max_iter=params['max_iter'], ptn_split=params['ptn_split'],
out_dir=p_out, out_prefix=prefix)
except:
logger.error('NO atlas estimated!')
logger.error(traceback.format_exc())
atlas = np.zeros_like(imgs[0])
w_bins = np.zeros((len(imgs), 1))
return atlas, w_bins
def load_data_ground_truth(params):
"""
:param params:
:return:
"""
p_data = os.path.join(params['in_path'], params['dataset'])
GT_atlas = gen_data.dataset_create_atlas(path_base=p_data)
GT_encoding = gen_data.dataset_load_weights(path_base=p_data)
......@@ -121,7 +167,24 @@ def load_data_ground_truth(params):
return GT_atlas, GT_img_rct
def expt_bacth_pipeline_alpe(GT_atlas, GT_img_rct, params):
def perform_experiment(expt_batch):
gt_atlas, gt_img_rct, i, imgs, params = expt_batch
init_atlas = alpe_init_atlas(imgs[0].shape, params['nb_lbs'],
params['init_tp'])
atlas, w_bins = alpe_pipe_atlas(i, imgs, init_atlas, params)
stat = alpe_compute_stat(gt_atlas, gt_img_rct, atlas, w_bins)
stat.update({'idx': i})
return stat
def expt_bacth_pipeline_alpe(gt_atlas, gt_img_rct, params):
"""
:param gt_atlas:
:param gt_img_rct:
:param params:
:return:
"""
tl_expt.check_exist_dirs_files(params)
params['res_path'] = tl_data.create_experiment_folder(params['out_path'],
params['name'])
......@@ -131,16 +194,24 @@ def expt_bacth_pipeline_alpe(GT_atlas, GT_img_rct, params):
p_data = os.path.join(params['in_path'], params['dataset'])
imgs = gen_data.dataset_load_images(params['sub_dataset'], p_data)
im_size = imgs[0].shape
df_stat = pd.DataFrame()
for i in range(params['nb_runs']):
init_atlas = alpe_init_atlas(im_size, params['nb_lbs'], params['init_tp'])
atlas, w_bins = alpe_pipe_atlas_learning_ptn_weights(i, imgs,
init_atlas, params)
stat = alpe_compute_stat(GT_atlas, GT_img_rct, atlas, w_bins)
mproc_pool = mproc.Pool(mproc.cpu_count())
expt_batch = [(gt_atlas, gt_img_rct, i, imgs, params)
for i in range(params['nb_runs'])]
l_stat = mproc_pool.map(perform_experiment, expt_batch)
for stat in l_stat:
df_stat = df_stat.append(stat, ignore_index=True)
df_stat.to_csv(os.path.join(params['res_path'], 'results.csv'))
df_stat = df_stat.set_index('idx')
df_stat.to_csv(os.path.join(params['res_path'], 'results.csv'))
# for i in range(params['nb_runs']):
# init_atlas = alpe_init_atlas(im_size, params['nb_lbs'], params['init_tp'])
# atlas, w_bins = alpe_pipe_atlas(i, imgs, init_atlas, params)
# stat = alpe_compute_stat(gt_atlas, gt_img_rct, atlas, w_bins)
# df_stat = df_stat.append(stat, ignore_index=True)
# df_stat.to_csv(os.path.join(params['res_path'], 'results.csv'))
logger.debug('statistic: \n{}'.format(df_stat.describe()))
f = open(p_res_stat, 'a')
......@@ -154,14 +225,70 @@ def expt_bacth_pipeline_alpe(GT_atlas, GT_img_rct, params):
# TODO: other datasets
def main():
# experiment_pipeline_alpe_showcase()
def params_extend_all_sub_datasets(l_params):
"""
:param l_params:
:return:
"""
l_params_ds = []
for p in l_params:
for ds in DEFAULT_SUB_DATASETS:
p_ds = p.copy()
p_ds.update({'sub_dataset': ds})
l_params_ds.append(p_ds)
return l_params_ds
def experiment_nb_classes(name='nb-Labels', params=None):
"""
:param name:
:param params:
:return:
"""
if params is None:
params = DEFAULT_PARAMS.copy()
params.update({'name': name})
gt_atlas, gt_img_rct = load_data_ground_truth(params)
l_params = []
for nb_lbs in range(2, 15):
p = params.copy()
p.update({'nb_lbs': nb_lbs})
l_params.append(p)
logger.debug('list params: {}'.format(len(l_params)))
l_params = params_extend_all_sub_datasets(l_params)
for p in l_params:
expt_bacth_pipeline_alpe(gt_atlas, gt_img_rct, p)
return
def experiments_all():
config = DEFAULT_PARAMS.copy()
experiment_nb_classes()
config.update({'ptn_split': False})
experiment_nb_classes(params=config)
config = DEFAULT_PARAMS.copy()
config.update({'init_tp': 'rnd'})
experiment_nb_classes()
config.update({'ptn_split': False})
experiment_nb_classes(params=config)
return
def experiments_test():
experiment_pipeline_alpe_showcase()
params = DEFAULT_PARAMS.copy()
GT_atlas, GT_img_rct = load_data_ground_truth(params)
expt_bacth_pipeline_alpe(GT_atlas, GT_img_rct, params)
return
return None
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
......@@ -170,9 +297,9 @@ if __name__ == "__main__":
# test_encoding(atlas, imgs, encoding)
# test_atlasLearning(atlas, imgs, encoding)
# test_simple_show_case()
# experiments_test()
main()
experiments_all()
logger.info('DONE')
plt.show()
\ No newline at end of file
......@@ -16,14 +16,12 @@ import src.ownUtils.toolDataIO as tl_data
jirka = False
if jirka:
DEFAULT_PATH_DATA = '/home/jirka/TEMP/'
DEFAULT_PATH_TEMP = '/home/jirka/TEMP/'
else:
# DEFAULT_PATH_DATA = '/datagrid/Medical/microscopy/drosophila_segmOvary/'
DEFAULT_PATH_DATA = '/datagrid/temporary/Medical/'
DEFAULT_PATH_TEMP = '/datagrid/temporary/Medical'
DEFAULT_DIR_APD = 'atomicPatternDictionary'
DEFAULT_PATH_APD = os.path.join(DEFAULT_PATH_DATA,DEFAULT_DIR_APD)
DEFAULT_PATH_APD = os.path.join(DEFAULT_PATH_DATA, DEFAULT_DIR_APD)
# DEFAULT_IM_SIZE = (512, 512)
# DEFAULT_IM_SIZE = (256, 256)
DEFAULT_IM_SIZE = (64, 64)
......@@ -46,7 +44,8 @@ def image_deform_elastic(im, coef=0.5, grid_size=(20, 20)):
:param grid_size: (int, int) is size of elastic grid for deformation
:return: np.array<w, h>
"""
logger.debug('deform image plane by elastic transform with grid {}'.format(grid_size))
logger.debug('deform image plane by elastic transform '
'with grid {}'.format(grid_size))
# logger.debug(im.shape)
rows, cols = np.meshgrid(np.linspace(0, im.shape[0], grid_size[0]),
np.linspace(0, im.shape[1], grid_size[1]))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment