Commit d011027e authored by Jiri Borovec's avatar Jiri Borovec

segm fail detect

parent d1904cf4
......@@ -110,7 +110,7 @@ class ExperimentAPD(tl_expt.Experiment):
name = '{}_{}'.format(params['name'], params['dataset'])
if not os.path.exists(params['out_path']):
super(ExperimentAPD, self).__init__(params, name)
super(self.__class__, self).__init__(params, name)
self.df_stat = pd.DataFrame()
self.p_stat = os.path.join(self.params.get('exp_path'), self.RESULTS_TXT)
......@@ -145,7 +145,7 @@ class ExperimentAPD(tl_expt.Experiment):
self.iter_n_var = it_var
self.iter_vals = it_vals
super(ExperimentAPD, self).run(gt)
super(self.__class__, self).run(gt)
def _perform(self):
""" perform experiment as sequence of iterated configurations """
......@@ -243,7 +243,7 @@ class ExperimentAPD_mp(ExperimentAPD):
:param params: {str: ...}
:param nb_jobs: int
super(ExperimentAPD_mp, self).__init__(params)
super(self.__class__, self).__init__(params)
self.nb_jobs = nb_jobs
def _perform_once(self, v):
......@@ -253,7 +253,7 @@ class ExperimentAPD_mp(ExperimentAPD):
self.params.set(self.iter_n_var, v)
stat = super(ExperimentAPD_mp, self)._perform_once(v)
stat = super(self.__class__, self)._perform_once(v)
return stat
def _perform_sequence(self):
This diff is collapsed.
script that walk ober all segmentation, compute some statistic
and then decide while the segmentation is likey to be correct of not
import os
import glob
import logging
import multiprocessing as mproc
import itertools
import shutil
# to suppress all visu, has to be on the beginning
import matplotlib
import numpy as np
from skimage import io
import matplotlib.pyplot as plt
import pandas as pd
logger = logging.getLogger(__name__)
PATH_BASE = '/datagrid/Medical/microscopy/drosophila/'
# PATH_SEGM = os.path.join(PATH_BASE, 'TEMPORARY/orig_segm')
PATH_SEGM = os.path.join(PATH_BASE, 'real_segmentations/orig_segm')
PATH_VISU = os.path.join(PATH_BASE, 'TEMPORARY/orig_visu')
# PATH_SEGM = os.path.join(PATH_BASE, 'real_segmentations/stage_4_segm')
# PATH_VISU = os.path.join(PATH_BASE, 'real_segmentations/stage_4_visu')
CSV_SEGM_GOOD = 'segm_good.csv'
CSV_SEGM_FAIL = 'segm_fail.csv'
FIG_STAT = 'stat_segm_labels.jpeg'
def labels_ration(p_seg):
""" compute radion among labels in segmentation as histogram
:param p_seg: str
:return: {int: float}
seg = io.imread(p_seg)
n_seg = os.path.basename(p_seg)
d_lb_hist = {lb: np.sum(seg == lb) / float(np.product(seg.shape) )
for lb in np.unique(seg)}
# the image borders
seg_border = np.concatenate((seg[:NB_IMG_CORNER, :].ravel(),
seg[-NB_IMG_CORNER:, :].ravel(),
seg[:, :NB_IMG_CORNER].ravel(),
seg[:, -NB_IMG_CORNER:].ravel()))
r_bg = np.sum(seg_border == 0) / float(seg_border.shape[0])
return {'name': n_seg,
'lb_hist': d_lb_hist,
'r_bg': r_bg}
def plot_histo_labels(d_hist, p_dir=''):
""" plot some simple histogram
:param d_hist: {int: float}
logger.debug('plotting results')
fig = plt.figure()
for lb in d_hist:
plt.plot(d_hist[lb], '+', label=str(lb))
plt.xlim([0, max(len(v) for v in d_hist.itervalues())])
plt.xlabel('image samples')
plt.ylabel('label cover')
if os.path.exists(p_dir):
fig.savefig(os.path.join(p_dir, FIG_STAT))
def read_make_hist(p_segs):
""" in parallel read all segmentation and compute individual histogram
:param p_segs: [str], paths to all segmentation
:return:[str, {int: float}] list or pairs with image name
and relative label histogram
nb_jobs = mproc.cpu_count()
logger.debug('run in %i threads...', nb_jobs)
mproc_pool = mproc.Pool(nb_jobs)
l_desc =, p_segs)
return l_desc
def merge_hist_stat(l_n_hist):
""" merge particular histograms per segmentation into one global per label
:param l_n_hist: [str, {int: float}] list or pairs with image name
and relative label histogram
:return: {int: [float]}, histogram per label over all images
logger.debug('merge partial results...')
l_hist = [l['lb_hist'] for l in l_n_hist]
lbs = itertools.chain(*[h.keys() for h in l_hist])
u_lbs = np.unique(lbs).tolist()
d_hist = {lb: [h[lb] for h in l_hist if lb in h]
for lb in u_lbs}
# join the foregrounds
d_hist['fg'] = []
for hist in l_hist:
d_hist['fg'].append(hist.get(1, 0) + hist.get(2, 0))
logger.debug('compute statistic...')
for lb, vals in d_hist.iteritems():'label %s with mean %f, median %f, std %f',
str(lb), np.mean(vals), np.median(vals), np.std(vals))
logger.debug(' -> count outliers: %i',
np.sum(abs(vals - np.median(vals)) > 3 * np.std(vals)))
return d_hist
def segm_decision(l_desc, d_hist):
""" according given rules decide weather the segmentation is good or not
:param l_desc: [{str, {int: float}}] list or pairs with image name
and relative label histogram
:param d_hist: {int: [float]}, histogram per label over all images
:return: [str], [str]
l_good, l_fail = [], []
fg_median, fg_std = np.median(d_hist['fg']), np.std(d_hist['fg'])
for i, desc in enumerate(l_desc):
fg = desc['lb_hist'].get(1, 0) + desc['lb_hist'].get(2, 0)
b_range = abs(fg - fg_median) <= 3 * fg_std
if desc['lb_hist'][0] < 0.85 and b_range and desc['r_bg'] > 0.95:
return l_good, l_fail
def export_results(p_dir, l_good, l_fail):
""" export the results into csv file
:param p_dir: str
:param l_good: [str], names of images
:param l_fail: [str], names of images
"""'export results as CSV files')
# with open(os.path.join(PATH_SEGM, 'segm_good.csv'), 'w') as f:
# f.writelines(['images'] + l_good)
# with open(os.path.join(PATH_SEGM, 'segm_fail.csv'), 'w') as f:
# f.writelines(['images'] + l_fail)
pd.DataFrame(['images'] + l_good).to_csv(
os.path.join(p_dir, CSV_SEGM_GOOD), index=False, header=False)
pd.DataFrame(['images'] + l_fail).to_csv(
os.path.join(p_dir, CSV_SEGM_FAIL), index=False, header=False)
def segm_detect_fails(p_dir=PATH_SEGM, im_ptn='*.png'):
""" make the statistic over all segmentation in given folder
and decide weather that are correct or fails
:param p_dir: str
:param im_ptn: str, pattern fro images
"""'FOLDER: "%s"', p_dir)
if not os.path.exists(p_dir):
raise Exception('folder "{}" dnot exist'.format(p_dir))
p_segs = glob.glob(os.path.join(p_dir, im_ptn))
logger.debug('found %i segmentation', len(p_segs))
l_desc = read_make_hist(p_segs)
d_hist = merge_hist_stat(l_desc)
# make the decision while segm is fine
l_good, l_fail = segm_decision(l_desc, d_hist)'number good %i and fails %i', len(l_good), len(l_fail))
export_results(p_dir, l_good, l_fail)
# show all samples
plot_histo_labels(d_hist, p_dir)
def filter_copy_visu(p_dir_seg=PATH_SEGM, p_dir_visu=PATH_VISU):
""" load csv file vith good and bad segmentation and in the visual folder
create subfolder for good and bad segm and copy relevant iimages there
:param p_dir_seg: str
:param p_dir_visu: str
logger.debug('segmentation: %s, visual: %s', p_dir_seg, p_dir_visu)
for n_csv in [CSV_SEGM_GOOD, CSV_SEGM_FAIL]:
logger.debug('reading "%s"', n_csv)
p_out = os.path.join(p_dir_visu, os.path.splitext(n_csv)[0])
if not os.path.exists(p_out):
df = pd.DataFrame.from_csv(os.path.join(p_dir_seg, n_csv),
logger.debug('copy %i images to "%s"', len(df), n_csv)
for n_img in df['images'].values.tolist():
shutil.copyfile(os.path.join(p_dir_visu, 'visu_' + n_img),
os.path.join(p_out, 'visu_' + n_img))
def main():
""" the main entry point """
# defaults run
for idx in range(1, 5):
p_dir_seg = os.path.join(PATH_BASE, 'real_segmentations/'
p_dir_visu = os.path.join(PATH_BASE, 'TEMPORARY/'
filter_copy_visu(p_dir_seg, p_dir_visu)
if __name__ == '__main__':
\ No newline at end of file
......@@ -109,6 +109,11 @@ def crop_images(imgs, v_lim=0.):
def threshold_image_fix(img):
""" treashold imahe with givem value in range (0, 1)
:param img:
return img_th
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment