run_experiment_segm_disc.py 19.7 KB
Newer Older
Jiri Borovec's avatar
Jiri Borovec committed
1
"""
Jiri Borovec's avatar
Jiri Borovec committed
2 3 4
segmentation on disc images with specific configuration
* perform the segmentation and export results
* load results and does visualisation
Jiri Borovec's avatar
Jiri Borovec committed
5

Jiri Borovec's avatar
Jiri Borovec committed
6
Example run: (no params)
Jiri Borovec's avatar
Jiri Borovec committed
7

Jiri Borovec's avatar
Jiri Borovec committed
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
>> python run_experiment_segm_disc.py \
    -in ~/Dropbox/Workspace/py_ImageProcessing/images \
    -out ~/Dropbox/Workspace/py_ImageProcessing/output \
    --dataset drosophila_disc --nb_jobs 2

>> python run_experiment_segm_disc.py \
    -in /datagrid/Medical/microscopy/drosophila/disc_images_types \
    -out /datagrid/Medical/microscopy/drosophila/TEMPORARY/segm_disc_pproc-gc-edge-weight \
    --dataset type_3 --visual 1

DONE:
* visualise prob weights for each class
* visualise edge weights per sperpixel pair
https://vcansimplify.wordpress.com/2014/07/06/scikit-image-rag-introduction/
http://scipy8.rssing.com/chan-20823059/all_p20.html
https://vcansimplify.wordpress.com/category/python-2/

Jiri Borovec's avatar
Jiri Borovec committed
25
Copyright (C) 2015-2016 Jiri Borovec <jiri.borovec@fel.cvut.cz>
Jiri Borovec's avatar
Jiri Borovec committed
26 27 28 29
"""


import os
Jiri Borovec's avatar
Jiri Borovec committed
30
import sys
Jiri Borovec's avatar
Jiri Borovec committed
31 32
import glob
import time
Jiri Borovec's avatar
Jiri Borovec committed
33 34
import gc
import argparse
Jiri Borovec's avatar
Jiri Borovec committed
35
import logging
Jiri Borovec's avatar
Jiri Borovec committed
36
import traceback
Jiri Borovec's avatar
Jiri Borovec committed
37
import json
Jiri Borovec's avatar
Jiri Borovec committed
38
import multiprocessing as mproc
39
from functools import partial
Jiri Borovec's avatar
Jiri Borovec committed
40

Jiri Borovec's avatar
Jiri Borovec committed
41
# to suppress all visu, has to be on the beginning
Jiri Borovec's avatar
Jiri Borovec committed
42 43
# from paramiko.logging22 import logger

44
import tqdm
Jiri Borovec's avatar
Jiri Borovec committed
45 46
import matplotlib
matplotlib.use('Agg')
Jiri Borovec's avatar
Jiri Borovec committed
47
import numpy as np
Jiri Borovec's avatar
Jiri Borovec committed
48 49
import matplotlib.pyplot as plt
from scipy import ndimage
Jiri Borovec's avatar
Jiri Borovec committed
50
from skimage import io, exposure, restoration, measure, draw, morphology
Jiri Borovec's avatar
Jiri Borovec committed
51 52 53
from skimage import segmentation as ski_segm

sys.path.append(os.path.abspath(os.path.join('..','..')))
Jiri Borovec's avatar
Jiri Borovec committed
54
import src.segmentation.pipelines as segm
Jiri Borovec's avatar
Jiri Borovec committed
55
import src.own_utils.tool_experiments as tl_expt
Jiri Borovec's avatar
Jiri Borovec committed
56

Jiri Borovec's avatar
Jiri Borovec committed
57

Jiri Borovec's avatar
Jiri Borovec committed
58 59
DATASET = 'orig'
PATH_DATA = '/datagrid/Medical/microscopy/drosophila/disc_types'
Jiri Borovec's avatar
Jiri Borovec committed
60 61 62 63 64
# PATH_RESULTS = '/datagrid/Medical/microscopy/drosophila/RESULTS'
PATH_RESULTS = '/datagrid/Medical/microscopy/drosophila/TEMPORARY'
PREFIX_VISU_SEGM = 'visu_segm_'
PREFIX_VISU_NORM = 'visu_norm_'
PREFIX_VISU_PIPELINE = 'visu_'
Jiri Borovec's avatar
Jiri Borovec committed
65 66
IMAGE_NAME = '*.png'
NB_THREADS = int(mproc.cpu_count() * .8)
Jiri Borovec's avatar
Jiri Borovec committed
67 68 69

DEFAULT_PARAMS = {
    'computer': os.uname(),
Jiri Borovec's avatar
Jiri Borovec committed
70
    'sp_size': 20,
Jiri Borovec's avatar
Jiri Borovec committed
71
    'sp_regul': 0.15,
Jiri Borovec's avatar
Jiri Borovec committed
72
    'gc_regul': 0.9, # 1.2
Jiri Borovec's avatar
Jiri Borovec committed
73
    'gc_edge_type': 'weight', # 'const', 'weight'
Jiri Borovec's avatar
Jiri Borovec committed
74
    'nb_labels': 3,
Jiri Borovec's avatar
Jiri Borovec committed
75 76
    'fts': {'clr': ['mean', 'std', 'eng']},
    # 'fts': {'clr': ['median', 'std', 'eng']},
Jiri Borovec's avatar
Jiri Borovec committed
77
    'clr': 'rgb',  # 'lab'
Jiri Borovec's avatar
Jiri Borovec committed
78 79
    'proba_type': 'quantiles', # 'GMM', 'quantiles', 'kmeans'
    # 'visu': True,
Jiri Borovec's avatar
Jiri Borovec committed
80
}
Jiri Borovec's avatar
Jiri Borovec committed
81

Jiri Borovec's avatar
Jiri Borovec committed
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113

def aparse_params():
    """
    SEE: https://docs.python.org/3/library/argparse.html
    :return: {str: str}, int
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-in', '--path_in', type=str, required=False,
                        help='path to the input directory', default=PATH_DATA)
    parser.add_argument('-out', '--path_out', type=str, required=False,
                        help='path to the output directory', default=PATH_RESULTS)
    parser.add_argument('--dataset', type=str, required=False,
                        help='name of the dataset to be segmented', default=DATASET)
    parser.add_argument('--nb_jobs', type=int, required=False, default=NB_THREADS,
                        help='number of processes in parallel')
    parser.add_argument('--visual', type=int, required=False, default=0,
                        help='whether di the visualisation')
    args = parser.parse_args()
    # args = vars(parser.parse_args())
    args.path_in = os.path.abspath(os.path.expanduser(args.path_in))
    assert os.path.exists(args.path_in), '%s' % args.path_in
    args.path_out = os.path.abspath(os.path.expanduser(args.path_out))
    assert os.path.exists(args.path_out), '%s' % args.path_out
    dict_paths = {
        'dataset_name': args.dataset,
        'path_in': os.path.join(args.path_in, args.dataset),
        'path_norm': os.path.join(args.path_out, args.dataset + '_norm'),
        'path_result': os.path.join(args.path_out, args.dataset + '_segm'),
        'path_visu': os.path.join(args.path_out, args.dataset + '_visu'),
    }
    assert os.path.exists(dict_paths['path_in']), '%s' % dict_paths['path_in']
    return dict_paths, args.nb_jobs, bool(args.visual)
Jiri Borovec's avatar
Jiri Borovec committed
114

Jiri Borovec's avatar
Jiri Borovec committed
115

Jiri Borovec's avatar
Jiri Borovec committed
116 117 118 119 120 121
def segments_sum_filled(seg):
    """ take each class and find smallest compact class

    :param seg: np.array<w, h>
    :return: [int]
    """
Jiri Borovec's avatar
Jiri Borovec committed
122 123 124 125
    # plt.figure()
    lb_sum = {}
    for i, lb in enumerate(np.unique(seg)):
        im = (seg == lb)
Jiri Borovec's avatar
Jiri Borovec committed
126
        im = morphology.binary_closing(im, morphology.disk(25))
Jiri Borovec's avatar
Jiri Borovec committed
127 128 129 130
        im = ndimage.binary_fill_holes(im)
        lb_sum[lb] = np.sum(im)
        # plt.subplot(1, 3, i+1), plt.imshow(im)
    lbs = sorted(lb_sum, key=lambda x: lb_sum[x], reverse=True)
Jiri Borovec's avatar
Jiri Borovec committed
131
    logging.debug('lb_sum: %s', repr(lb_sum))
Jiri Borovec's avatar
Jiri Borovec committed
132 133 134 135
    # plt.show()
    return lbs


Jiri Borovec's avatar
Jiri Borovec committed
136
def segment_values(img, seg):
Jiri Borovec's avatar
Jiri Borovec committed
137
    """ sort segments according mean image value
Jiri Borovec's avatar
Jiri Borovec committed
138 139 140 141 142 143 144

    :param img:
    :param seg:
    :return:
    """
    img_vec = img.reshape(-1, img.shape[-1])
    seg_vec = seg.reshape(-1)
Jiri Borovec's avatar
Jiri Borovec committed
145
    label_val = {}
Jiri Borovec's avatar
Jiri Borovec committed
146 147
    for i, lb in enumerate(np.unique(seg_vec)):
        im = img_vec[(seg_vec == lb)]
Jiri Borovec's avatar
Jiri Borovec committed
148 149 150 151
        label_val[lb] = np.median(im)
    labels = sorted(label_val, key=lambda x: label_val[x], reverse=True)
    logging.debug('lb_sum: %s', repr(label_val))
    return labels
Jiri Borovec's avatar
Jiri Borovec committed
152 153


Jiri Borovec's avatar
Jiri Borovec committed
154
def estimate_lut(img, seg):
Jiri Borovec's avatar
Jiri Borovec committed
155 156 157 158 159 160
    """ estimate relabeling LUT

    :param img:
    :param seg:
    :return:
    """
Jiri Borovec's avatar
Jiri Borovec committed
161 162 163 164
    # labels = segments_sum_filled(seg)
    labels = segment_values(img, seg)
    logging.debug('labels: %s & seg in range '
                 '%i:%i', repr(labels), np.min(seg), np.max(seg))
Jiri Borovec's avatar
Jiri Borovec committed
165
    lut = range(np.max(seg) + 1)
Jiri Borovec's avatar
Jiri Borovec committed
166
    for i, lb in enumerate(labels):
Jiri Borovec's avatar
Jiri Borovec committed
167 168 169 170
        lut[lb] = i
    return lut


Jiri Borovec's avatar
Jiri Borovec committed
171
def visual_pair_orig_segm(dict_paths, img, seg, img_name):
Jiri Borovec's avatar
Jiri Borovec committed
172 173
    """ visualise pair of original image and segm

Jiri Borovec's avatar
Jiri Borovec committed
174
    :param dict_paths:
Jiri Borovec's avatar
Jiri Borovec committed
175 176
    :param img:
    :param seg:
Jiri Borovec's avatar
Jiri Borovec committed
177
    :param img_name:
Jiri Borovec's avatar
Jiri Borovec committed
178 179
    """
    fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(24, 12))
Jiri Borovec's avatar
Jiri Borovec committed
180
    ax[0].set_title('original image')
Jiri Borovec's avatar
Jiri Borovec committed
181 182
    ax[0].imshow(img), ax[0].axis('off')
    ax[0].contour(seg, linewidth=2, cmap=plt.cm.Reds)
Jiri Borovec's avatar
Jiri Borovec committed
183
    ax[1].set_title('resulting segmentation')
Jiri Borovec's avatar
Jiri Borovec committed
184 185
    ax[1].imshow(seg), ax[1].axis('off')
    # ax[0].imshow(seg_sml), plt.axis('off')
Jiri Borovec's avatar
Jiri Borovec committed
186 187
    path_fig = os.path.join(dict_paths['path_visu'], PREFIX_VISU_SEGM + img_name)
    fig.savefig(path_fig, bbox_inches='tight')
Jiri Borovec's avatar
Jiri Borovec committed
188
    plt.close(fig)
Jiri Borovec's avatar
Jiri Borovec committed
189 190


Jiri Borovec's avatar
Jiri Borovec committed
191
def visual_pipeline(dict_paths, img, im_norm, seg_raw, seg, img_name):
Jiri Borovec's avatar
Jiri Borovec committed
192 193
    """ visualise complete pipeline process

Jiri Borovec's avatar
Jiri Borovec committed
194 195 196
    :param seg_raw:
    :param im_norm:
    :param dict_paths:
Jiri Borovec's avatar
Jiri Borovec committed
197 198
    :param img:
    :param seg:
Jiri Borovec's avatar
Jiri Borovec committed
199
    :param img_name:
Jiri Borovec's avatar
Jiri Borovec committed
200 201 202 203 204 205 206 207 208 209 210 211 212
    """
    fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(24, 20))
    ax[0, 0].set_title('original image')
    ax[0, 0].imshow(img), ax[0, 0].axis('off')
    ax[0, 0].contour(seg, linewidth=2, cmap=plt.cm.Reds)
    ax[1, 0].set_title('normalised image')
    ax[1, 0].imshow(im_norm), ax[1, 0].axis('off')
    ax[1, 0].contour(seg_raw, linewidth=2, cmap=plt.cm.Reds)
    ax[0, 1].set_title('resulting segmentation')
    ax[0, 1].imshow(seg), ax[0, 1].axis('off')
    ax[1, 1].set_title('raw segmentation')
    ax[1, 1].imshow(seg_raw), ax[1, 1].axis('off')
    # ax[0].imshow(seg_sml), plt.axis('off')
Jiri Borovec's avatar
Jiri Borovec committed
213
    path_fig = os.path.join(dict_paths['path_visu'], PREFIX_VISU_PIPELINE + img_name)
Jiri Borovec's avatar
Jiri Borovec committed
214
    fig.savefig(path_fig, bbox_inches='tight')
Jiri Borovec's avatar
Jiri Borovec committed
215
    plt.close(fig)
Jiri Borovec's avatar
Jiri Borovec committed
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230


def draw_img_histogram(ax, img):
    ax.hist((img[:, :, 0], img[:, :, 1], img[:, :, 2]), bins=48,
            color=('red', 'green', 'blue'))
    # for i, clr in enumerate(list('rgb')):
    #     im_clr = img[:, :, i]
    #     count, bins = np.histogram(im_clr, bins=128)
    #     ax.plot(bins[:-1], count / float(np.product(im_clr.shape)))
    ax.set_xlabel('color values')
    ax.set_xlim([0, np.max(img)])
    ax.set_ylabel('[%]')
    ax.grid()


Jiri Borovec's avatar
Jiri Borovec committed
231
def visual_pair_orig_norm(dict_paths, img, img_norm, img_name):
Jiri Borovec's avatar
Jiri Borovec committed
232 233
    """ visualise pair of original image and segm

Jiri Borovec's avatar
Jiri Borovec committed
234
    :param dict_paths:
Jiri Borovec's avatar
Jiri Borovec committed
235
    :param img:
Jiri Borovec's avatar
Jiri Borovec committed
236 237
    :param img_norm:
    :param img_name:
Jiri Borovec's avatar
Jiri Borovec committed
238 239 240 241 242 243
    """
    fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(20, 15))
    ax[0, 0].set_title('original image')
    ax[0, 0].imshow(img), ax[0, 0].axis('off')
    draw_img_histogram(ax[1, 0], img)
    ax[0, 1].set_title('normalise image')
Jiri Borovec's avatar
Jiri Borovec committed
244 245 246 247 248
    ax[0, 1].imshow(img_norm), ax[0, 1].axis('off')
    draw_img_histogram(ax[1, 1], img_norm)
    path_fig = os.path.join(dict_paths['path_visu'], PREFIX_VISU_NORM +
                         img_name.replace('.png', '.pdf'))
    fig.savefig(path_fig, bbox_inches='tight')
Jiri Borovec's avatar
Jiri Borovec committed
249
    plt.close(fig)
Jiri Borovec's avatar
Jiri Borovec committed
250 251 252 253 254 255 256 257


def preprocessing_image(img):
    """ simple image pre-processing and nralisation

    :param img: np.array<h, w, 3>
    :return: np.array<h, w, 3>
    """
Jiri Borovec's avatar
Jiri Borovec committed
258 259 260 261
    logging.debug('perform the PRE-processing...')
    p_low = np.percentile(img, 2)
    p_high = np.percentile(img, 90)
    img_new = exposure.rescale_intensity(img, in_range=(p_low, p_high))
Jiri Borovec's avatar
Jiri Borovec committed
262
    # http://scikit-image.org/docs/dev/auto_examples/plot_denoise.html
Jiri Borovec's avatar
Jiri Borovec committed
263 264 265
    # PROBLEM with v0.13
    # img_new = restoration.denoise_bilateral(img_new, sigma_range=0.1,
    #                                         sigma_spatial=150)
Jiri Borovec's avatar
Jiri Borovec committed
266 267 268 269 270
    # http://scikit-image.org/docs/dev/auto_examples/plot_equalize.html
    # img_new = exposure.equalize_adapthist(img_new, clip_limit=0.05)
    return img_new


Jiri Borovec's avatar
Jiri Borovec committed
271
def postprocessing_segm_3cls(img, seg, label_disc=1, label_act=2):
Jiri Borovec's avatar
Jiri Borovec committed
272 273 274 275 276 277
    """ postprocessing and some morpho operations

    :param img: np.array<h, w, 3>
    :param seg: np.array<h, w>
    :return: np.array<h, w>
    """
Jiri Borovec's avatar
Jiri Borovec committed
278
    logging.info('perform the POST-processing...')
Jiri Borovec's avatar
Jiri Borovec committed
279 280
    # normalise and order labels
    seg -= seg.min()
Jiri Borovec's avatar
Jiri Borovec committed
281 282
    lut = estimate_lut(img, seg)
    logging.debug('LUT: {}'.format(lut))
Jiri Borovec's avatar
Jiri Borovec committed
283 284
    seg = np.asarray(lut)[seg]

Jiri Borovec's avatar
Jiri Borovec committed
285 286 287
    # DISCS
    seg_fg = seg >= label_disc
    seg_act = seg == label_act
Jiri Borovec's avatar
Jiri Borovec committed
288

Jiri Borovec's avatar
Jiri Borovec committed
289
    # correct the foreground layer
Jiri Borovec's avatar
Jiri Borovec committed
290
    seg_fg = morphology.binary_closing(seg_fg, morphology.disk(15))
Jiri Borovec's avatar
Jiri Borovec committed
291
    seg_fg = ndimage.binary_fill_holes(seg_fg)
Jiri Borovec's avatar
Jiri Borovec committed
292
    # # filter activation
Jiri Borovec's avatar
Jiri Borovec committed
293
    seg_act = morphology.binary_opening(seg_act, morphology.disk(9))
Jiri Borovec's avatar
Jiri Borovec committed
294 295 296

    # take centered elements
    sg, _ = ndimage.label(seg_fg)
Jiri Borovec's avatar
Jiri Borovec committed
297 298 299 300 301 302 303 304 305 306
    measure_seg = measure.regionprops(sg)
    lim_size = 0.1 * np.product(seg.shape)
    dist = [np.sum((0.5 * np.asarray(seg.shape) - np.asarray(m.centroid)) ** 2)
            for m in measure_seg if m.area > lim_size]
    if len(dist) == 0:
        return seg
    label_sel = measure_seg[dist.index(min(dist))].label
    seg_sel = (sg == label_sel)

    # TODO: active contours ??
Jiri Borovec's avatar
Jiri Borovec committed
307
    # seg_sel = segm_active_contours(img, seg_sel)
Jiri Borovec's avatar
Jiri Borovec committed
308 309

    seg_new = np.zeros_like(seg)
Jiri Borovec's avatar
Jiri Borovec committed
310 311
    seg_new[seg_sel] = label_disc
    seg_new[np.logical_and(seg_sel, seg_act)] = label_act
Jiri Borovec's avatar
Jiri Borovec committed
312 313 314
    return seg_new


Jiri Borovec's avatar
Jiri Borovec committed
315
def segm_active_contours(img, seg_fg):
Jiri Borovec's avatar
Jiri Borovec committed
316 317 318 319
    """
    http://scikit-image.org/docs/dev/api/skimage.segmentation.html#skimage.segmentation.active_contour

    :param img:
Jiri Borovec's avatar
Jiri Borovec committed
320 321
    :param seg_fg: np.array<h, w>
    :return: np.array<h, w>
Jiri Borovec's avatar
Jiri Borovec committed
322
    """
Jiri Borovec's avatar
Jiri Borovec committed
323
    logging.debug('perform active contours...')
Jiri Borovec's avatar
Jiri Borovec committed
324 325 326 327 328 329 330 331
    contours = measure.find_contours(seg_fg, 0)
    snake = ski_segm.active_contour(img, contours[0], max_iterations=999)
    mask = np.zeros_like(seg_fg)
    rr, cc = draw.polygon(snake[:, 0], snake[:, 1])
    mask[rr, cc] = 1
    return mask


Jiri Borovec's avatar
Jiri Borovec committed
332
def export_debug_images(path_out, img_name, dict_debug_imgs):
Jiri Borovec's avatar
Jiri Borovec committed
333 334
    """

Jiri Borovec's avatar
Jiri Borovec committed
335 336 337
    :param path_out:
    :param img_name:
    :param dict_debug_imgs:
Jiri Borovec's avatar
Jiri Borovec committed
338
    """
Jiri Borovec's avatar
Jiri Borovec committed
339 340 341 342 343 344 345 346 347 348 349 350
    path_debug = os.path.join(path_out, 'debug_' + os.path.splitext(img_name)[0])
    logging.debug('debug images: "%s"', repr(path_debug))
    if not os.path.exists(path_debug):
        os.mkdir(path_debug)
    for k in ['slic_mean', 'img_graph_edges', 'img_graph_segm']:
        # if k in dict_debug_imgs: # in case of GC const edges, there is no edge image
        io.imsave(os.path.join(path_debug, k + '.png'), dict_debug_imgs[k])
    img_contour = ski_segm.mark_boundaries(dict_debug_imgs['img'],
                                           dict_debug_imgs['slic'], (1, 0, 0))
    io.imsave(os.path.join(path_debug, 'slic_contour.png'), img_contour)
    for i, im_uc in enumerate(dict_debug_imgs['imgs_unary_cost']):
        io.imsave(os.path.join(path_debug, 'im_unary_cost_{}.png'.format(i)), im_uc)
Jiri Borovec's avatar
Jiri Borovec committed
351 352


353
def segment_image(path_img, params, dict_paths):
Jiri Borovec's avatar
Jiri Borovec committed
354 355 356
    """ segment individual image

    :param params: {str: ...}
357
    :param path_img: str
Jiri Borovec's avatar
Jiri Borovec committed
358
    :param dict_paths: {str: ...}
Jiri Borovec's avatar
Jiri Borovec committed
359 360
    :param visu: bool
    """
Jiri Borovec's avatar
Jiri Borovec committed
361
    img_name = os.path.basename(path_img)
362
    img_raw = io.imread(path_img)
Jiri Borovec's avatar
Jiri Borovec committed
363

Jiri Borovec's avatar
Jiri Borovec committed
364
    img = preprocessing_image(img_raw)
Jiri Borovec's avatar
Jiri Borovec committed
365 366 367 368 369 370 371 372
    io.imsave(os.path.join(dict_paths['path_norm'], img_name), img)

    logging.info('perform the SEGMENTATION...')
    logging.debug('image name: {}'.format(img_name))
    logging.debug('img values range: %f - %f', np.min(img), np.max(img))
    dict_debug_imgs = None
    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
        dict_debug_imgs = dict()
Jiri Borovec's avatar
Jiri Borovec committed
373

Jiri Borovec's avatar
Jiri Borovec committed
374
    t = time.time()
Jiri Borovec's avatar
Jiri Borovec committed
375
    seg_raw = segm.pipe_clr2d_spx_fts_gmm_gc(img, nb_cls=params['nb_labels'],
Jiri Borovec's avatar
Jiri Borovec committed
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
                                             clr_space=params['clr'],
                                             sp_size=params['sp_size'],
                                             sp_regul=params['sp_regul'],
                                             proba_type=params['proba_type'],
                                             gc_regul=params['gc_regul'],
                                             dict_features=params['fts'],
                                             gc_edge_type=params['gc_edge_type'],
                                             dict_debug_imgs=dict_debug_imgs)
    logging.info('execution time [s]: %f', (time.time()-t))

    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
        dict_debug_imgs['img'] = img_raw
        dict_debug_imgs['img_norm'] = img
        export_debug_images(dict_paths['path_result'], img_name, dict_debug_imgs)

    seg = postprocessing_segm_3cls(img, seg_raw)

    path_seg = os.path.join(dict_paths['path_result'], img_name)
Jiri Borovec's avatar
Jiri Borovec committed
394
    # io.imsave(p_img, np.array((seg == lbs[-1]) * 255, dtype=np.uint8))
Jiri Borovec's avatar
Jiri Borovec committed
395 396 397 398
    logging.debug('image save to "%s"', path_seg)
    io.imsave(path_seg, np.array(seg, dtype=np.uint8))
    visual_pipeline(dict_paths, img_raw, img, seg_raw, seg, img_name)
    gc.collect(), time.sleep(1)
Jiri Borovec's avatar
Jiri Borovec committed
399 400


Jiri Borovec's avatar
Jiri Borovec committed
401 402
def segment_image_folder(dict_paths, params=DEFAULT_PARAMS,
                         im_pattern=IMAGE_NAME, nb_jobs=1):
Jiri Borovec's avatar
Jiri Borovec committed
403 404
    """ segment complete image folder

Jiri Borovec's avatar
Jiri Borovec committed
405
    :param dict_paths: {str: ...}
Jiri Borovec's avatar
Jiri Borovec committed
406 407 408
    :param params: {str: ...}
    :param im_pattern: str
    """
Jiri Borovec's avatar
Jiri Borovec committed
409

Jiri Borovec's avatar
Jiri Borovec committed
410 411
    check_create_dirs(dict_paths, ['path_in'],
                      ['path_norm', 'path_result', 'path_visu'])
Jiri Borovec's avatar
Jiri Borovec committed
412
    with open(os.path.join(dict_paths['path_result'], 'config.txt'), 'w') as f:
Jiri Borovec's avatar
Jiri Borovec committed
413
        f.write(tl_expt.string_dict(params))
Jiri Borovec's avatar
Jiri Borovec committed
414
    json.dump(params, open(os.path.join(dict_paths['path_result'], 'config.json'), 'w'))
415
    paths_img = sorted(glob.glob(os.path.join(dict_paths['path_in'], im_pattern)))
Jiri Borovec's avatar
Jiri Borovec committed
416
    logging.info('found %i images', len(paths_img))
417 418 419 420

    tqdm_bar = tqdm.tqdm(total=len(paths_img))
    wrapper_segment_image = partial(segment_image,
                                    params=params, dict_paths=dict_paths)
Jiri Borovec's avatar
Jiri Borovec committed
421 422

    if nb_jobs > 1:
Jiri Borovec's avatar
Jiri Borovec committed
423
        logging.debug('wrapper_segment_image in %i threads', nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
424
        mproc_pool = mproc.Pool(nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
425
        for x in mproc_pool.imap_unordered(wrapper_segment_image, paths_img):
426
            tqdm_bar.update()
Jiri Borovec's avatar
Jiri Borovec committed
427 428 429
        mproc_pool.close()
        mproc_pool.join()
    else:
430 431 432
        for path_img in paths_img:
            segment_image(path_img, params, dict_paths)
            tqdm_bar.update()
Jiri Borovec's avatar
Jiri Borovec committed
433 434


Jiri Borovec's avatar
Jiri Borovec committed
435
def load_visual_pair_orig_segm(name_img, dict_paths):
Jiri Borovec's avatar
Jiri Borovec committed
436 437
    """ visualise segmentation together with original images, version fro mproc

Jiri Borovec's avatar
Jiri Borovec committed
438
    :param mp_set: (params, n_img)
Jiri Borovec's avatar
Jiri Borovec committed
439
    """
Jiri Borovec's avatar
Jiri Borovec committed
440 441 442 443
    logging.debug('visual: "%s"', name_img)
    path_img = os.path.join(dict_paths['path_in'], name_img)
    path_seg = os.path.join(dict_paths['path_result'], name_img)
    if not os.path.exists(path_img) or not os.path.exists(path_seg):
Jiri Borovec's avatar
Jiri Borovec committed
444
        logging.warning('image of segment. does not exist')
Jiri Borovec's avatar
Jiri Borovec committed
445
        return
Jiri Borovec's avatar
Jiri Borovec committed
446 447 448
    img = io.imread(path_img)
    seg = io.imread(path_seg)
    visual_pair_orig_segm(dict_paths, img, seg, name_img)
Jiri Borovec's avatar
Jiri Borovec committed
449 450


Jiri Borovec's avatar
Jiri Borovec committed
451
def check_create_dirs(dict_paths, l_required, l_create):
Jiri Borovec's avatar
Jiri Borovec committed
452 453
    """ check path existence and create some new

Jiri Borovec's avatar
Jiri Borovec committed
454 455 456
    :param dict_paths: {str: str}
    :param l_required: [str] required dirs
    :param l_create: [str] create new dirs
Jiri Borovec's avatar
Jiri Borovec committed
457
    """
Jiri Borovec's avatar
Jiri Borovec committed
458 459 460 461 462 463
    for n in l_create:
        if not os.path.exists(dict_paths[n]):
            os.mkdir(dict_paths[n])
    for n in l_required + l_create:
        logging.info('"%s" dir: (%s) <- %s', n, os.path.exists(dict_paths[n]), dict_paths[n])
    if any([not os.path.exists(dict_paths[n]) for n in l_required + l_create]):
Jiri Borovec's avatar
Jiri Borovec committed
464 465 466
        raise Exception('one or more paths do not exist')


Jiri Borovec's avatar
Jiri Borovec committed
467
def visu_folder_imgs_segm(dict_paths, im_pattern=IMAGE_NAME, nb_jobs=NB_THREADS):
Jiri Borovec's avatar
Jiri Borovec committed
468 469 470 471 472
    """ visualise complete folder

    :param params: {str: ...}
    :param im_pattern: str
    """
Jiri Borovec's avatar
Jiri Borovec committed
473
    check_create_dirs(dict_paths, ['path_in', 'path_result'], ['path_visu'])
Jiri Borovec's avatar
Jiri Borovec committed
474 475 476 477 478 479
    path_segs = glob.glob(os.path.join(dict_paths['path_result'], im_pattern))
    logging.info('found %i segmentation', len(path_segs))
    logging.debug('segmentation: %s...', repr(path_segs[:3]))

    tqdm_bar = tqdm.tqdm(total=len(path_segs))
    wrapper_visual = partial(load_visual_pair_orig_segm, dict_paths=dict_paths)
Jiri Borovec's avatar
Jiri Borovec committed
480

Jiri Borovec's avatar
Jiri Borovec committed
481 482 483 484
    mproc_pool = mproc.Pool(nb_jobs)
    for x in mproc_pool.imap_unordered(wrapper_visual,
                                       (os.path.basename(p) for p in path_segs)):
        tqdm_bar.update()
Jiri Borovec's avatar
Jiri Borovec committed
485 486
    mproc_pool.close()
    mproc_pool.join()
Jiri Borovec's avatar
Jiri Borovec committed
487 488


Jiri Borovec's avatar
Jiri Borovec committed
489
def norm_raw_image(name_img, dict_paths):
Jiri Borovec's avatar
Jiri Borovec committed
490
    """ load input raw image, normalise it and then save it
Jiri Borovec's avatar
Jiri Borovec committed
491

Jiri Borovec's avatar
Jiri Borovec committed
492 493
    :param name_img:
    :param dict_paths:
Jiri Borovec's avatar
Jiri Borovec committed
494 495
    """
    try:
Jiri Borovec's avatar
Jiri Borovec committed
496
        img = io.imread(os.path.join(dict_paths['path_in'], name_img))
Jiri Borovec's avatar
Jiri Borovec committed
497
        im_norm = preprocessing_image(img)
Jiri Borovec's avatar
Jiri Borovec committed
498 499
        # io.imsave(os.path.join(dict_paths['path_norm'], name_img), im_norm)
        visual_pair_orig_norm(dict_paths, img, im_norm, name_img)
Jiri Borovec's avatar
Jiri Borovec committed
500
    except:
Jiri Borovec's avatar
Jiri Borovec committed
501 502
        logging.error('image normal fail: "%s"', name_img)
        logging.error(traceback.format_exc())
Jiri Borovec's avatar
Jiri Borovec committed
503 504


Jiri Borovec's avatar
Jiri Borovec committed
505
def norm_folder_raw_imgs(dict_paths, nb_jobs=NB_THREADS, im_pattern=IMAGE_NAME):
Jiri Borovec's avatar
Jiri Borovec committed
506 507
    """ visualise complete folder

Jiri Borovec's avatar
Jiri Borovec committed
508
    :param dict_paths: {str: ...}
Jiri Borovec's avatar
Jiri Borovec committed
509 510
    :param im_pattern: str
    """
Jiri Borovec's avatar
Jiri Borovec committed
511
    check_create_dirs(dict_paths, ['path_in'], ['path_norm', 'path_visu'])
Jiri Borovec's avatar
Jiri Borovec committed
512 513 514 515 516
    path_imgs = glob.glob(os.path.join(dict_paths['path_in'], im_pattern))
    logging.info('found %i images', len(path_imgs))

    tqdm_bar = tqdm.tqdm(total=len(path_imgs))
    wrapper_norm_raw_image = partial(norm_raw_image, dict_paths=dict_paths)
Jiri Borovec's avatar
Jiri Borovec committed
517

Jiri Borovec's avatar
Jiri Borovec committed
518 519 520 521
    mproc_pool = mproc.Pool(nb_jobs)
    for x in mproc_pool.imap_unordered(wrapper_norm_raw_image,
                                       (os.path.basename(p) for p in path_imgs)):
        tqdm_bar.update()
Jiri Borovec's avatar
Jiri Borovec committed
522 523 524 525
    mproc_pool.close()
    mproc_pool.join()


Jiri Borovec's avatar
Jiri Borovec committed
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
# def main_segment_disc_types():
#     """ perform the segmentation pipeline with exporting normed images
#     and post visualisation over all types of discs """
#     for idx in range(1, 5):
#         n_dataset = 'type_{}'.format(idx)
#         logging.info('segmenting: %s', n_dataset)
#         d_paths = DEFAULT_PATHS.copy()
#         d_paths.update({
#             'dataset_name': n_dataset,
#             'path_in': os.path.join(PATH_DATA, n_dataset),
#             'path_norm': os.path.join(PATH_TEMP, n_dataset + '_norm'),
#             'path_result': os.path.join(PATH_RESULTS, n_dataset + '_segm'),
#             'path_visu': os.path.join(PATH_TEMP, n_dataset + '_visu'),})
#         norm_folder_raw_imgs(dict_paths=d_paths)
#         segment_image_folder(dict_paths=d_paths, nb_jobs=NB_THREADS)
#         visu_folder_imgs_segm(dict_paths=d_paths)
Jiri Borovec's avatar
Jiri Borovec committed
542 543


Jiri Borovec's avatar
Jiri Borovec committed
544
def main():
Jiri Borovec's avatar
Jiri Borovec committed
545 546 547 548 549 550
    logging.basicConfig(level=logging.INFO)
    logging.info('running...')

    dict_paths, nb_jobs, b_visu = aparse_params()
    if b_visu:
        logging.getLogger().setLevel(logging.DEBUG)
Jiri Borovec's avatar
Jiri Borovec committed
551

Jiri Borovec's avatar
Jiri Borovec committed
552
    segment_image_folder(dict_paths, nb_jobs=nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
553

Jiri Borovec's avatar
Jiri Borovec committed
554 555 556
    # if b_visu:
    #     norm_folder_raw_imgs(dict_paths, nb_jobs)
    #     visu_folder_imgs_segm(dict_paths, nb_jobs=nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
557

Jiri Borovec's avatar
Jiri Borovec committed
558
    logging.info('DONE')
Jiri Borovec's avatar
Jiri Borovec committed
559 560 561 562


if __name__ == "__main__":
    main()