run_preproc_prob_segm.py 8 KB
Newer Older
Jiri Borovec's avatar
Jiri Borovec committed
1 2 3
"""
run experiments with Atomic Learning Pattern Encoding

Jiri Borovec's avatar
Jiri Borovec committed
4
Copyright (C) 2015-2016 Jiri Borovec <jiri.borovec@fel.cvut.cz>
Jiri Borovec's avatar
Jiri Borovec committed
5
"""
Jiri Borovec's avatar
Jiri Borovec committed
6

Jiri Borovec's avatar
Jiri Borovec committed
7 8

import os
9 10
import gc
import time
Jiri Borovec's avatar
Jiri Borovec committed
11 12 13
import logging
import multiprocessing as mproc

Jiri Borovec's avatar
Jiri Borovec committed
14 15 16
# to suppres all visu, has to be on the beginning
import matplotlib
matplotlib.use('Agg')
Jiri Borovec's avatar
Jiri Borovec committed
17
import numpy as np
Jiri Borovec's avatar
Jiri Borovec committed
18 19
from skimage import filters, morphology, transform

Jiri Borovec's avatar
Jiri Borovec committed
20
import dataset_utils as gen_data
Jiri Borovec's avatar
Jiri Borovec committed
21

Jiri Borovec's avatar
Jiri Borovec committed
22 23
logger = logging.getLogger(__name__)

Jiri Borovec's avatar
Jiri Borovec committed
24
# PATH_DATA = '/b_jirka/b_jirka/TEMP/APD_real_data'
Jiri Borovec's avatar
Jiri Borovec committed
25
DEFAULT_PATH_DATA = '/datagrid/Medical/microscopy/drosophila/'
Jiri Borovec's avatar
Jiri Borovec committed
26 27 28

# REAL_DATASET_NAME = '1000_ims'
# IMAGE_PATTERN = '*_exp'
Jiri Borovec's avatar
Jiri Borovec committed
29 30
# REAL_DATASET_NAME = '1000_images_improved'
# IMAGE_PATTERN = '*_seg_de'
Jiri Borovec's avatar
Jiri Borovec committed
31

Jiri Borovec's avatar
Jiri Borovec committed
32 33
REAL_DATASET_NAME = 'type_1_segm_reg'
IMAGE_PATTERN = '*'
34

Jiri Borovec's avatar
Jiri Borovec committed
35
DEFAULT_PARAMS = {
Jiri Borovec's avatar
Jiri Borovec committed
36
    # 'computer': os.uname(),
Jiri Borovec's avatar
Jiri Borovec committed
37
    'path_in': os.path.join(DEFAULT_PATH_DATA, 'RESULTS', REAL_DATASET_NAME),
Jiri Borovec's avatar
Jiri Borovec committed
38
    'path_out': os.path.join(DEFAULT_PATH_DATA, 'TEMPORARY'),
Jiri Borovec's avatar
Jiri Borovec committed
39 40
    # 'binary': ['fix', 'otsu', 'adapt'],
    'binary': ['3cls'],
Jiri Borovec's avatar
Jiri Borovec committed
41
}
42 43
BINARY_POSIX = '_binary'
NB_THREADS = int(mproc.cpu_count() * 0.7)
Jiri Borovec's avatar
Jiri Borovec committed
44
IMAGE_BINARY_THRESHOLD = 0.9
Jiri Borovec's avatar
Jiri Borovec committed
45 46 47 48 49 50 51 52 53 54


def extend_images(imgs):
    """ the method requres images of equal dims
    so if there is different image extend them by zeros

    :param imgs: [np.array<w, h>]
    :return: [np.array<w, h>]
    """
    im_sizes = [im.shape for im in imgs]
Jiri Borovec's avatar
Jiri Borovec committed
55
    logger.info('different image sizes: {}'.format(set(im_sizes)))
Jiri Borovec's avatar
Jiri Borovec committed
56 57 58 59 60
    if len(set(im_sizes)) == 1:
        return imgs
    w_max = max([w for w, h in im_sizes])
    h_max = max([h for w, h in im_sizes])
    m_size = (w_max, h_max)
61
    logger.debug('max image size: {}'.format(m_size))
Jiri Borovec's avatar
Jiri Borovec committed
62 63 64 65 66 67 68 69 70 71
    for i, im in enumerate(imgs):
        if im.shape == m_size:
            continue
        logger.debug('.. extend img {} dims {} -> {}'.format(i, im.shape, m_size))
        im_new = np.zeros(m_size)
        im_new[:im.sahpe[0], :im.sahpe[1]] = im
        imgs[i] = im_new
    return imgs


72
def find_borders(im_mean, v_lim=0.):
Jiri Borovec's avatar
Jiri Borovec committed
73 74
    """ find from sided the rows and col where the image set is zero

Jiri Borovec's avatar
Jiri Borovec committed
75 76
    :param v_lim:
    :param im_mean:
Jiri Borovec's avatar
Jiri Borovec committed
77 78 79
    :return:
    """
    border = {}
Jiri Borovec's avatar
Jiri Borovec committed
80
    logger.info('crop value threshold: {}'.format(v_lim))
81 82 83 84 85 86 87 88 89
    for i in range(len(im_mean.shape)):
        vals = im_mean.mean(axis=i)
        vals[vals < v_lim] = 0
        idx = np.nonzero(vals)[0]
        if len(idx) > 0:
            border[i] = min(idx), max(idx)
        else:
            logger.warning('nothing to chose in axis: {}'.format(i))
            border[i] = 0, -1
Jiri Borovec's avatar
Jiri Borovec committed
90 91 92
    return border


93
def crop_images(imgs, v_lim=0.):
Jiri Borovec's avatar
Jiri Borovec committed
94 95 96 97 98
    """ try to cut out image rows and colums that are useless

    :param imgs: [np.array<w, h>]
    :return: [np.array<w, h>]
    """
99 100 101
    logger.info('perform image crop...')
    im_mean = np.array(imgs).mean(axis=0)
    border = find_borders(im_mean, v_lim)
Jiri Borovec's avatar
Jiri Borovec committed
102
    logger.info('image crop limits: {}'.format(border))
Jiri Borovec's avatar
Jiri Borovec committed
103
    imgs_crop = [None] * len(imgs)
Jiri Borovec's avatar
Jiri Borovec committed
104
    logger.info('crop all {} images'.format(len(imgs)))
Jiri Borovec's avatar
Jiri Borovec committed
105 106 107 108 109 110 111
    for i, im in enumerate(imgs):
        # the sum dim is perpendicular to cut dim
        imgs_crop[i] = im[border[1][0]:border[1][1], border[0][0]:border[0][1]]
    logger.debug('image crop finished')
    return imgs_crop


Jiri Borovec's avatar
Jiri Borovec committed
112
def threshold_image_fix(idx_img):
Jiri Borovec's avatar
Jiri Borovec committed
113 114 115 116 117
    """ treashold imahe with givem value in range (0, 1)

    :param img:
    :return:
    """
Jiri Borovec's avatar
Jiri Borovec committed
118
    idx, img = idx_img
Jiri Borovec's avatar
Jiri Borovec committed
119
    img_th = img > IMAGE_BINARY_THRESHOLD
Jiri Borovec's avatar
Jiri Borovec committed
120
    return idx, img_th
Jiri Borovec's avatar
Jiri Borovec committed
121 122


Jiri Borovec's avatar
Jiri Borovec committed
123 124
def threshold_image_otsu(idx_img):
    idx, img = idx_img
Jiri Borovec's avatar
Jiri Borovec committed
125 126 127
    th = filters.threshold_otsu(img[img > 0])
    logger.debug('threshold with val: {}'.format(th))
    img_th = img > th
Jiri Borovec's avatar
Jiri Borovec committed
128
    return idx, img_th
Jiri Borovec's avatar
Jiri Borovec committed
129 130


Jiri Borovec's avatar
Jiri Borovec committed
131 132
def threshold_image_adapt(idx_img):
    idx, img = idx_img
133 134
    img_th = filters.threshold_adaptive(img, 450)
    selem = morphology.disk(12)
Jiri Borovec's avatar
Jiri Borovec committed
135
    img_th = morphology.opening(img_th, selem)
Jiri Borovec's avatar
Jiri Borovec committed
136
    return idx, img_th
Jiri Borovec's avatar
Jiri Borovec committed
137 138


Jiri Borovec's avatar
Jiri Borovec committed
139 140
def threshold_image_3cls_gene(idx_img):
    idx, img = idx_img
Jiri Borovec's avatar
Jiri Borovec committed
141
    img_th = img >= (2 / 3.)
Jiri Borovec's avatar
Jiri Borovec committed
142
    return idx, img_th
Jiri Borovec's avatar
Jiri Borovec committed
143 144


Jiri Borovec's avatar
Jiri Borovec committed
145 146
def threshold_image_3cls_disc(idx_img):
    idx, img = idx_img
Jiri Borovec's avatar
Jiri Borovec committed
147
    img_th = img >= (1 / 3.)
Jiri Borovec's avatar
Jiri Borovec committed
148
    return idx, img_th
Jiri Borovec's avatar
Jiri Borovec committed
149 150


Jiri Borovec's avatar
Jiri Borovec committed
151 152
def scale_image(idx_img):
    idx, img = idx_img
Jiri Borovec's avatar
Jiri Borovec committed
153
    img_scale = transform.rescale(img, scale=0.5, order=0)
Jiri Borovec's avatar
Jiri Borovec committed
154
    return idx, img_scale
Jiri Borovec's avatar
Jiri Borovec committed
155 156 157


def apply_images(imgs, func, nb_jobs=NB_THREADS):
Jiri Borovec's avatar
Jiri Borovec committed
158 159
    """ threshold images by specific level

Jiri Borovec's avatar
Jiri Borovec committed
160 161
    :param nb_jobs:
    :param func:
Jiri Borovec's avatar
Jiri Borovec committed
162 163 164
    :param imgs: [np.array<w, h>]
    :return: [np.array<w, h>]
    """
Jiri Borovec's avatar
Jiri Borovec committed
165 166 167 168
    logger.info('apply function "%s on %i images...', func.__name__, len(imgs))
    if nb_jobs > 1:
        logger.info('running in %i threads...', nb_jobs)
        mproc_pool = mproc.Pool(nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
169
        idx_imgs = mproc_pool.map(func, enumerate(imgs))
Jiri Borovec's avatar
Jiri Borovec committed
170 171 172 173
        mproc_pool.close()
        mproc_pool.join()
    else:
        logger.info('running single thread')
Jiri Borovec's avatar
Jiri Borovec committed
174 175
        idx_imgs = [func(im) for im in enumerate(imgs)]
    imgs_final = [img for _, img in sorted(idx_imgs)]
Jiri Borovec's avatar
Jiri Borovec committed
176 177 178
    return imgs_final


Jiri Borovec's avatar
Jiri Borovec committed
179
def binarize_all(params=DEFAULT_PARAMS, im_pattern=IMAGE_PATTERN, nb_jobs=NB_THREADS):
Jiri Borovec's avatar
Jiri Borovec committed
180 181 182 183
    """ run all experiments

    :return:
    """
Jiri Borovec's avatar
Jiri Borovec committed
184 185
    imgs, names = gen_data.dataset_load_images(params['path_in'], im_pattern=im_pattern,
                                               nb_jobs=nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
186
    logger.info('loaded {} images of size {}'.format(len(imgs), imgs[0].shape))
Jiri Borovec's avatar
Jiri Borovec committed
187
    imgs = extend_images(imgs)
188 189
    imgs = crop_images(imgs, 1e-3)
    gc.collect(), time.sleep(1)
Jiri Borovec's avatar
Jiri Borovec committed
190

Jiri Borovec's avatar
Jiri Borovec committed
191 192
    path_export = os.path.join(params['path_out'],
                               os.path.basename(params['path_in']) + BINARY_POSIX)
Jiri Borovec's avatar
Jiri Borovec committed
193 194 195
    if not os.path.exists(path_export):
        os.mkdir(path_export)
    logger.debug('exporting path: %s', path_export)
Jiri Borovec's avatar
Jiri Borovec committed
196

Jiri Borovec's avatar
Jiri Borovec committed
197
    if 'fix' in params['binary']:
Jiri Borovec's avatar
Jiri Borovec committed
198 199 200 201
        imgs_th = apply_images(imgs, threshold_image_fix)
        name_dir = 'binary-fix_{}'.format(IMAGE_BINARY_THRESHOLD)
        gen_data.dataset_export_images(os.path.join(path_export, name_dir),
                                       imgs_th, names, nb_jobs=nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
202
        gc.collect(), time.sleep(1)
Jiri Borovec's avatar
Jiri Borovec committed
203

Jiri Borovec's avatar
Jiri Borovec committed
204
    if 'otsu' in params['binary']:
Jiri Borovec's avatar
Jiri Borovec committed
205 206 207
        imgs_th = apply_images(imgs, threshold_image_otsu)
        gen_data.dataset_export_images(os.path.join(path_export, 'binary-otsu'),
                                       imgs_th, names, nb_jobs=nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
208 209 210
        gc.collect(), time.sleep(1)

    if 'adapt' in params['binary']:
Jiri Borovec's avatar
Jiri Borovec committed
211 212 213 214
        imgs_th = apply_images(imgs, threshold_image_adapt)
        gen_data.dataset_export_images(os.path.join(path_export, 'binary-adapt'),
                                       imgs_th, names, nb_jobs=nb_jobs)
        gc.collect(), time.sleep(1)
Jiri Borovec's avatar
Jiri Borovec committed
215

Jiri Borovec's avatar
Jiri Borovec committed
216
    if '3cls' in params['binary']:
Jiri Borovec's avatar
Jiri Borovec committed
217 218 219 220 221 222 223 224 225 226 227 228 229 230
        logger.info('detecting just last class in 3-class segmentation')
        imgs_th = apply_images(imgs, threshold_image_3cls_gene)
        gen_data.dataset_export_images(os.path.join(path_export, 'gene'),
                                       imgs_th, names, nb_jobs=nb_jobs)
        gc.collect(), time.sleep(1)
        logger.info('scaling images by 0.5')
        imgs_scale = apply_images(imgs_th, scale_image)
        gen_data.dataset_export_images(os.path.join(path_export, 'gene_small'),
                                       imgs_scale, names, nb_jobs=nb_jobs)
        logger.info('scaling images by 0.5')
        imgs_scale = apply_images(imgs_scale, scale_image)
        gen_data.dataset_export_images(os.path.join(path_export, 'gene_ssmall'),
                                       imgs_scale, names, nb_jobs=nb_jobs)
        gc.collect(), time.sleep(1)
Jiri Borovec's avatar
Jiri Borovec committed
231

Jiri Borovec's avatar
Jiri Borovec committed
232 233 234 235 236
        logger.info('detecting foreground class in 3-class segmentation')
        imgs_th = apply_images(imgs, threshold_image_3cls_disc)
        gen_data.dataset_export_images(os.path.join(path_export, 'disc'),
                                       imgs_th, names, nb_jobs=nb_jobs)
        gc.collect(), time.sleep(1)
Jiri Borovec's avatar
Jiri Borovec committed
237 238


Jiri Borovec's avatar
Jiri Borovec committed
239
def main():
Jiri Borovec's avatar
Jiri Borovec committed
240 241 242
    logging.basicConfig(level=logging.DEBUG)
    logger.info('running...')

Jiri Borovec's avatar
Jiri Borovec committed
243 244 245
    params = DEFAULT_PARAMS.copy()
    datasets = ['type_{}_segm_reg'.format(i) for i in range(1, 5)]
    for ds in datasets:
Jiri Borovec's avatar
Jiri Borovec committed
246 247
        params['path_in'] = os.path.join(DEFAULT_PATH_DATA, 'RESULTS',
                                         REAL_DATASET_NAME, ds)
Jiri Borovec's avatar
Jiri Borovec committed
248 249 250
        binarize_all(params)

    logger.info('DONE')
Jiri Borovec's avatar
Jiri Borovec committed
251

Jiri Borovec's avatar
Jiri Borovec committed
252 253 254

if __name__ == "__main__":
    main()