Commit 8475cb0c authored by Jiri Borovec's avatar Jiri Borovec

update, STA expt on real

parent fd42e976
......@@ -139,7 +139,7 @@ def create_clean_folder(path_dir):
:param path_dir: str, path
:return: str
"""
assert os.path.exists(os.path.dirname(path_dir))
assert os.path.exists(os.path.dirname(path_dir)), os.path.dirname(path_dir)
logger.info('create clean folder "%s"', path_dir)
if os.path.exists(path_dir):
shutil.rmtree(path_dir)
......@@ -467,7 +467,7 @@ def dataset_load_images(path_dir, im_pattern='*', nb_spls=None, nb_jobs=1):
:return: [np.array], [str]
"""
logger.debug('loading folder (%s) <- "%s"', os.path.exists(path_dir), path_dir)
assert os.path.exists(path_dir)
assert os.path.exists(path_dir), path_dir
paths_img = find_images(path_dir, im_pattern)
paths_img = sorted(paths_img)[:nb_spls]
logger.debug('number samples %i in dataset "%s"', len(paths_img),
......@@ -496,7 +496,7 @@ def dataset_load_images(path_dir, im_pattern='*', nb_spls=None, nb_jobs=1):
def load_image(path_img):
assert os.path.exists(path_img)
assert os.path.exists(path_img), path_img
n_img, img_ext = os.path.splitext(os.path.basename(path_img))
if img_ext in ['.tif', '.tiff']:
im = libtiff.TiffFile(path_img).get_tiff_array()
......@@ -531,7 +531,7 @@ def dataset_load_weights(path_base, name_csv=CSV_NAME_WEIGHTS):
return np.array(encoding)
def dataset_create_atlas(path_base, name=DIR_NAME_DICTIONARY, img_temp_name='pattern_*'):
def dataset_compose_atlas(path_base, name=DIR_NAME_DICTIONARY, img_temp_name='pattern_*'):
""" load all independent patterns and compose them into single m-label atlas
:param name: str, name of dataset
......
......@@ -342,7 +342,7 @@ def alpe_pipe_atlas_learning_ptn_weights(imgs, init_atlas=None, init_weights=Non
:param out_dir: str, path to the results directory
:return: np.array<w, h>, np.array<nb_imgs, nb_lbs>
"""
logging.info('compute an Atlas and weights for %i images...', len(imgs))
logging.debug('compute an Atlas and weights for %i images...', len(imgs))
assert len(imgs) >= 0
if logging.getLogger().getEffectiveLevel()==logging.DEBUG:
if not os.path.exists(out_dir):
......
......@@ -51,9 +51,9 @@ PATH_RESULTS = '/datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_AP
DEFAULT_PARAMS = {
'computer': os.uname(),
'nb_samples': None,
'init_tp': 'rnd', # msc. rnd
'max_iter': 150,
'gc_regul': 1e-2,
'init_tp': 'msc1', # msc. rnd
'max_iter': 250,
'gc_regul': 0.,
'nb_labels': 20,
'nb_runs': NB_THREADS, # 500
'gc_reinit': True,
......@@ -75,9 +75,9 @@ SYNTH_PARAMS.update({
})
SYNTH_PTN_RANGE = {
'atomicPatternDictionary_00': range(5),
'atomicPatternDictionary_v0': range(2, 14, 1),
'atomicPatternDictionary_v0': range(3, 15, 1),
'atomicPatternDictionary_v1': range(5, 20, 1),
'atomicPatternDictionary_v2': range(10, 40, 2),
'atomicPatternDictionary_v2': range(9, 40, 2),
'atomicPatternDictionary_v3': range(10, 40, 2),
'atomicPatternDictionary3D_v0': range(2, 14, 1),
'atomicPatternDictionary3D_v1': range(6, 30, 2),
......@@ -86,6 +86,7 @@ SYNTH_PTN_RANGE = {
REAL_DATASET_NAME = '1000_images_improved_binary'
# REAL_SUB_DATASETS = ['binary-fix', 'binary-otsu', 'binary-adapt']
NB_PATTERNS_REAL = [5, 8, 10, 12, 14, 16, 18, 20, 25, 30, 40, 60]
REAL_SUB_DATASETS = [
# 'gene',
# 'gene_small',
......@@ -183,7 +184,7 @@ class ExperimentAPD(tl_expt.Experiment):
:param params: {str: ...}, parameter settings
"""
self.gt_atlas = gen_data.dataset_create_atlas(self.params.get('path_in'))
self.gt_atlas = gen_data.dataset_compose_atlas(self.params.get('path_in'))
gt_encoding = gen_data.dataset_load_weights(self.params.get('path_in'))
self.gt_img_rct = ptn_dict.reconstruct_samples(self.gt_atlas, gt_encoding)
......@@ -297,6 +298,10 @@ class ExperimentAPD(tl_expt.Experiment):
imgs_gt = self.gt_img_rct[:len(imgs_rct)]
diff = np.asarray(imgs_gt) - np.asarray(imgs_rct)
stat['reconstruct_diff'] = np.sum(abs(diff)) / float(np.prod(diff.shape))
elif hasattr(self, 'imgs') and imgs_rct is not None:
imgs = self.imgs[:len(imgs_rct)]
diff = np.asarray(imgs) - np.asarray(imgs_rct)
stat['reconstruct_diff'] = np.sum(abs(diff)) / float(np.prod(diff.shape))
return stat
def _evaluate(self):
......
......@@ -3,32 +3,6 @@
#cd ~/Dropbox/Workspace/py_ImageProcessing/src/atm_ptn_dict/
#source ~/vEnv/bin/activate
# STATE-OF-THE-ART methods
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v0 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v2 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v0 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-3D-sta
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-3D-sta
# OUR method
python run_experiment_apd_apdl.py \
......@@ -45,4 +19,31 @@ python run_experiment_apd_apdl.py \
python run_experiment_apd_apdl.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
\ No newline at end of file
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_synth
# STATE-OF-THE-ART methods - SPCA, ICA, DL, NMF, APDL
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v0 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_synth
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_synth
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v2 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_synth
python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v3 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_synth
# python run_experiment_apd_all.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v0 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD
# python run_experiment_apd_all.py \
# -in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary3D_v1 \
# -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD \
# --nb_jobs 1
\ No newline at end of file
......@@ -23,6 +23,7 @@ def initialise_atlas_random(im_size, max_lb):
"""
logging.debug('initialise atlas %s as random labeling', repr(im_size))
nb_lbs = max_lb + 1
np.random.seed() # reinit seed to have random samples even in the same time
im = np.random.randint(1, nb_lbs, im_size)
return np.array(im, dtype=np.int)
......@@ -37,6 +38,7 @@ def initialise_atlas_mosaic(im_size, max_lb, coef=1.):
"""
logging.debug('initialise atlas %s as grid labeling', repr(im_size))
nb_lbs = int(max_lb * coef)
np.random.seed() # reinit seed to have random samples even in the same time
block = np.ones(np.ceil(im_size / np.array(nb_lbs, dtype=np.float)))
vec = range(1, nb_lbs + 1) * int(np.ceil(coef))
logging.debug('block size is %s', repr(block.shape))
......
......@@ -22,6 +22,7 @@ import os
import glob
import logging
import gc
import sys
import time
import multiprocessing as mproc
from functools import partial
......@@ -33,7 +34,10 @@ import matplotlib.pyplot as plt
import dataset_utils as gen_data
import run_apd_reconstruction as r_reconst
sys.path.append(os.path.abspath(os.path.join('..','..'))) # Add path to root
import src.segmentation.tool_superpixels as tl_spx
CONNECT_PATTERNS = True
NB_THREADS = int(mproc.cpu_count() * .9)
NAME_CONFIG = 'config.json'
PREFIX_ATLAS = 'atlas_'
......@@ -70,7 +74,7 @@ def compute_merged_reconst_diff(ptn_comb, dict_params, path_out, atlas, img_name
def hierarchical_clustering_merge_patterns(dict_params, path_out, img_names,
atlas, nb_jobs=NB_THREADS):
atlas, nb_jobs=NB_THREADS, connect_ptns=CONNECT_PATTERNS):
""" using hierarchical clustering merge pattern pair and return partial results
:param dict_params: {str: ...}
......@@ -82,8 +86,11 @@ def hierarchical_clustering_merge_patterns(dict_params, path_out, img_names,
"""
labels = sorted(np.unique(atlas).tolist())
# generate combinations as list as skipping the 0 assuming on first position
ptn_combines = [(labels[i], labels[j])
for i in range(1, len(labels)) for j in range(1, i)]
if connect_ptns:
_, ptn_combines = tl_spx.make_graph_segm_connect2d_conn4(atlas)
else:
ptn_combines = [(labels[i], labels[j])
for i in range(1, len(labels)) for j in range(1, i)]
assert len(ptn_combines) > 0 and \
not any(len(set(ptn)) == 1 for ptn in ptn_combines)
# parallel compute reconstructions
......@@ -168,7 +175,7 @@ def sequence_hierarchical_clustering(dict_params, path_out, img_names, atlas,
def plot_reconst_diff(path_csv):
""" plot trhe recost diff from PD and expoert to a figure
""" plot the reconst. diff from PD and expoert to a figure
:param path_csv: str
"""
......@@ -177,6 +184,7 @@ def plot_reconst_diff(path_csv):
fig = plt.figure()
df_diff.plot(ax=fig.gca(), title='Reconstruction diff. for estimated atlases')
fig.gca().set_ylabel('reconst. diff [%]')
fig.gca().grid()
path_fig = os.path.join(os.path.dirname(path_csv), FIG_RECONT_DIFF)
fig.savefig(path_fig)
......@@ -184,6 +192,7 @@ def plot_reconst_diff(path_csv):
def process_experiment(path_expt, nb_jobs=NB_THREADS):
""" process complete folder with experiment
:param nb_jobs: int
:param path_expt: str
"""
logging.info('Experiment folder: \n "%s"', path_expt)
......
......@@ -4,10 +4,15 @@ run experiments with Stat-of-the-art methods
Example run:
>> nohup python experiments_sta.py > ~/Medical-temp/experiments_APD-sta/nohup.log &
>> python run_experiment_apd_sta.py \
>> python run_experiment_apd_all.py \
-in /datagrid/Medical/microscopy/drosophila/synthetic_data/atomicPatternDictionary_v1 \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD-sta
>> python run_experiment_apd_all.py --type real \
-in /datagrid/Medical/microscopy/drosophila/TEMPORARY/type_1_segm_reg_binary \
-out /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_real \
--dataset gene_ssmall
Copyright (C) 2015-2016 Jiri Borovec <jiri.borovec@fel.cvut.cz>
"""
......@@ -35,6 +40,8 @@ import run_experiment_apd_apdl as expt_apdl
SYNTH_PARAMS = expt_apd.SYNTH_PARAMS
SYNTH_SUB_DATASETS = expt_apd.SYNTH_SUB_DATASETS
SYNTH_PTN_RANGE = expt_apd.SYNTH_PTN_RANGE
REAL_PARAMS = expt_apd.REAL_PARAMS
NB_PATTERNS_REAL = expt_apd.NB_PATTERNS_REAL
class ExperimentLinearCombineBase(expt_apd.ExperimentAPD):
......@@ -274,14 +281,48 @@ def experiments_synthetic(params=SYNTH_PARAMS):
if params['nb_jobs'] <= 1:
cls_expt = METHODS_BASE[m]
tqdm_bar = tqdm.tqdm(total=len(l_params))
for params in l_params:
expt = cls_expt(params)
for param in l_params:
param['method'] = m
expt = cls_expt(param)
expt.run(iter_var='nb_labels', iter_vals=ptn_range)
tqdm_bar.update(1)
del expt
gc.collect(), time.sleep(1)
def experiments_real(params=REAL_PARAMS):
""" run all experiments
:param nb_jobs: int
:param params: {str: value}
"""
arg_params = expt_apd.parse_params(params)
logging.info('PARAMS: \n%s', '\n'.join(['"{}": \n\t {}'.format(k, v)
for k, v in arg_params.iteritems()]))
params.update(arg_params)
if not 'method' in params:
params['method'] = METHODS.keys()
l_params = [params]
if isinstance(params['dataset'], list):
l_params = expt_apd.extend_list_params(l_params, 'dataset', params['dataset'])
logging.debug('list params: %i', len(l_params))
tqdm_bar = tqdm.tqdm(total=len(l_params))
for m in params['method']:
cls_expt = METHODS[m]
if params['nb_jobs'] <= 1:
cls_expt = METHODS_BASE[m]
tqdm_bar = tqdm.tqdm(total=len(l_params))
for param in l_params:
param['method'] = m
expt = cls_expt(param)
expt.run(gt=False, iter_var='nb_labels', iter_vals=NB_PATTERNS_REAL)
tqdm_bar.update(1)
del expt
gc.collect(), time.sleep(1)
def main():
""" main_real entry point """
logging.basicConfig(level=logging.INFO)
......@@ -292,8 +333,8 @@ def main():
arg_params = expt_apd.parse_params(SYNTH_PARAMS)
if arg_params['type'] == 'synth':
experiments_synthetic()
else:
logging.error('not implemented yet')
elif arg_params['type'] == 'real':
experiments_real()
logging.info('DONE')
# plt.show()
......
......@@ -45,6 +45,24 @@ SYNTH_PARAMS = expt_apd.SYNTH_PARAMS
SYNTH_SUB_DATASETS = expt_apd.SYNTH_SUB_DATASETS
SYNTH_PTN_RANGE = expt_apd.SYNTH_PTN_RANGE
REAL_PARAMS = expt_apd.REAL_PARAMS
NB_PATTERNS_REAL = expt_apd.NB_PATTERNS_REAL
DICT_ATLAS_INIT = {
'msc': ptn_dict.initialise_atlas_mosaic,
'msc1': partial(ptn_dict.initialise_atlas_mosaic, coef=1.5),
'msc2': partial(ptn_dict.initialise_atlas_mosaic, coef=2),
'rnd': ptn_dict.initialise_atlas_random,
'GT': None, # init by Ground Truth, require GT atlas
'GTd': None, # init by deformed Ground Truth, require GT atlas
# 'OWS': None,
}
# SIMPLE RUN
INIT_TYPES = ['GT', 'GTd']
GRAPHCUT_REGUL = [0.0, 1e-15, 1e-3]
# COMPLEX RUN
# INIT_TYPES = DICT_ATLAS_INIT.keys()
# GRAPHCUT_REGUL = [0., 0e-12, 1e-9, 1e-6, 1e-3, 1e-1, 1.0]
def test_simple_show_case():
......@@ -88,7 +106,7 @@ def experiment_pipeline_alpe_showcase(path_out):
:param path_out: str
:return:
"""
atlas = gen_data.dataset_create_atlas(expt_apd.SYNTH_PATH_APD)
atlas = gen_data.dataset_compose_atlas(expt_apd.SYNTH_PATH_APD)
# plt.imshow(atlas)
path_in = os.path.join(expt_apd.SYNTH_PATH_APD, gen_data.NAME_DATASET)
......@@ -107,14 +125,6 @@ def experiment_pipeline_alpe_showcase(path_out):
return atlas, w_bins
DICT_ATLAS_INIT = {
'msc': ptn_dict.initialise_atlas_mosaic,
'msc1': partial(ptn_dict.initialise_atlas_mosaic, coef=1.5),
'msc2': partial(ptn_dict.initialise_atlas_mosaic, coef=2),
'rnd': ptn_dict.initialise_atlas_random,
}
class ExperimentAPDL_base(expt_apd.ExperimentAPD):
"""
the main real experiment or our Atlas Learning Pattern Encoding
......@@ -130,7 +140,15 @@ class ExperimentAPDL_base(expt_apd.ExperimentAPD):
im_size = self.imgs[0].shape
assert init_tp in DICT_ATLAS_INIT
fn_init_atlas = DICT_ATLAS_INIT[init_tp]
init_atlas = fn_init_atlas(im_size, nb_lbs)
if fn_init_atlas is not None:
init_atlas = fn_init_atlas(im_size, nb_lbs)
elif init_tp == 'GT':
assert hasattr(self, 'gt_atlas')
init_atlas = np.remainder(self.gt_atlas, nb_lbs)
elif init_tp == 'GTd':
assert hasattr(self, 'gt_atlas')
init_atlas = np.remainder(self.gt_atlas, nb_lbs)
init_atlas = ptn_dict.initialise_atlas_deform_original(init_atlas)
return init_atlas
def _estimate_atlas(self, v):
......@@ -140,7 +158,7 @@ class ExperimentAPDL_base(expt_apd.ExperimentAPD):
:param init_atlas: np.array<w, h>
:return: np.array, np.array
"""
logging.info(' -> estimate atlas...')
logging.debug(' -> estimate atlas...')
self.params[self.iter_var_name] = v
logging.debug('PARAMS: %s', repr(self.params))
init_atlas = self._init_atlas(self.params['nb_labels'], self.params['init_tp'])
......@@ -212,13 +230,6 @@ def experiments_test(dict_params=SYNTH_PARAMS):
expt_p.run(iter_var='case', iter_vals=range(params['nb_runs']))
# INIT_TYPES = ['msc', 'rnd']
GRAPHCUT_REGUL = [0.0, 1e-15, 1e-3]
INIT_TYPES = DICT_ATLAS_INIT.keys()
# GRAPHCUT_REGUL = [0., 0e-12, 1e-9, 1e-6, 1e-3, 1e-1, 1.0]
NB_PATTERNS_REAL = [5, 8, 10, 12, 14, 16, 18, 20, 25, 30, 40, 60]
def experiments_synthetic(params=SYNTH_PARAMS):
""" run all experiments
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment