dataset_utils.py 22.7 KB
Newer Older
Jiri Borovec's avatar
Jiri Borovec committed
1
"""
Jiri Borovec's avatar
Jiri Borovec committed
2
The basic module for generating synthetic images and also loading / exporting
Jiri Borovec's avatar
Jiri Borovec committed
3

Jiri Borovec's avatar
Jiri Borovec committed
4
Copyright (C) 2015-2016 Jiri Borovec <jiri.borovec@fel.cvut.cz>
Jiri Borovec's avatar
Jiri Borovec committed
5 6
"""

Jiri Borovec's avatar
Jiri Borovec committed
7 8 9
import os
import random
import glob
Jiri Borovec's avatar
Jiri Borovec committed
10
import logging
Jiri Borovec's avatar
Jiri Borovec committed
11
import itertools
Jiri Borovec's avatar
Jiri Borovec committed
12
import multiprocessing as mproc
Jiri Borovec's avatar
Jiri Borovec committed
13 14
from functools import partial
import shutil
Jiri Borovec's avatar
Jiri Borovec committed
15

Jiri Borovec's avatar
Jiri Borovec committed
16 17 18 19
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import ndimage
Jiri Borovec's avatar
Jiri Borovec committed
20
from skimage import draw, transform, filters
Jiri Borovec's avatar
Jiri Borovec committed
21
from PIL import Image
Jiri Borovec's avatar
Jiri Borovec committed
22 23
import libtiff
import tqdm
Jiri Borovec's avatar
Jiri Borovec committed
24

Jiri Borovec's avatar
Jiri Borovec committed
25
logger = logging.getLogger(__name__)
Jiri Borovec's avatar
Jiri Borovec committed
26

Jiri Borovec's avatar
Jiri Borovec committed
27
NB_THREADS = mproc.cpu_count()
Jiri Borovec's avatar
Jiri Borovec committed
28 29 30 31
IMAGE_SIZE_2D = (128, 128)
IMAGE_SIZE_3D = (16, 128, 128)
NB_ATM_PATTERNS = 9
NB_SAMPLES = 50
Jiri Borovec's avatar
Jiri Borovec committed
32
RND_PATTERN_OCCLUSION = 0.25
Jiri Borovec's avatar
Jiri Borovec committed
33
IMAGE_POSIXS = ['.png', '.tif', '.tiff']
Jiri Borovec's avatar
Jiri Borovec committed
34 35
IMAGE_PATTERN = 'pattern_{:03d}'
SEGM_PATTERN = 'sample_{:05d}'
Jiri Borovec's avatar
Jiri Borovec committed
36
BLOCK_NB_LOAD_IMAGES = 50
Jiri Borovec's avatar
Jiri Borovec committed
37 38 39
DIR_NAME_DICTIONARY = 'dictionary'
CSV_NAME_WEIGHTS = 'combination.csv'
NAME_DATASET = 'datasetBinary_raw'
Jiri Borovec's avatar
Jiri Borovec committed
40
COLUMN_NAME = 'ptn_{:02d}'
Jiri Borovec's avatar
Jiri Borovec committed
41 42


Jiri Borovec's avatar
Jiri Borovec committed
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
def create_elastic_deform_2d(im_size, coef=0.5, grid_size=(20, 20)):
    rows, cols = np.meshgrid(np.linspace(0, im_size[0], grid_size[0]),
                             np.linspace(0, im_size[1], grid_size[1]))
    mesh_src = np.dstack([cols.flat, rows.flat])[0]
    # logger.debug(src)
    mesh_dst = mesh_src.copy()
    for i in range(2):
        rnd = np.random.random((mesh_src.shape[0], 1)) - 0.5
        mesh_dst[:, i] += rnd[:, 0] * (im_size[i] / grid_size[i] * coef)
    mesh_dst = filters.gaussian_filter(mesh_dst, 0.1)
    # logger.debug(dst)
    tform = transform.PiecewiseAffineTransform()
    tform.estimate(mesh_src, mesh_dst)
    return tform


Jiri Borovec's avatar
Jiri Borovec committed
59 60
def image_deform_elastic(im, coef=0.5, grid_size=(20, 20)):
    """ deform an image bu elastic transform in size of specific regular grid
Jiri Borovec's avatar
Jiri Borovec committed
61

Jiri Borovec's avatar
Jiri Borovec committed
62 63 64 65 66
    :param im: np.array<w, h>
    :param coef: float, a param describing the how much it is deformed (0 = None)
    :param grid_size: (int, int) is size of elastic grid for deformation
    :return: np.array<w, h>
    """
Jiri Borovec's avatar
Jiri Borovec committed
67 68
    logger.debug('deform image plane by elastic transform with grid %s',
                 repr(grid_size))
Jiri Borovec's avatar
Jiri Borovec committed
69
    # logger.debug(im.shape)
Jiri Borovec's avatar
Jiri Borovec committed
70 71 72 73 74 75 76 77 78 79
    im_size = im.shape[-2:]
    tform = create_elastic_deform_2d(im_size, coef, grid_size)
    if im.ndim == 2:
        img = transform.warp(im, tform, output_shape=im_size, order=0,
                             cval=im[0, 0])
    elif im.ndim == 3:
        im_stack = [transform.warp(im[i], tform, output_shape=im_size,
                                   order=0, cval=im[0, 0, 0])
                    for i in range(im.shape[0])]
        img = np.array(im_stack)
Jiri Borovec's avatar
Jiri Borovec committed
80 81 82 83
    img = np.array(255 * img, dtype=np.int8)
    return img


Jiri Borovec's avatar
Jiri Borovec committed
84 85 86 87 88 89 90 91 92 93 94 95 96
def generate_rand_center_radius(img, ratio):
    center, radius = [0] * img.ndim, [0] * img.ndim
    for i in range(img.ndim):
        size = img.shape[i]
        center[i] = random.randint(int(1.5 * ratio * size),
                              int((1. - 1.5 * ratio) * size))
        radius[i] = random.randint(int(0.25 * ratio * size),
                              int(1. * ratio * size))
        radius[i] += 0.03 * size
    return center, radius


def draw_ellipse(img, ratio=0.1, clr=255):
Jiri Borovec's avatar
Jiri Borovec committed
97
    """ draw an ellipse to image plane with specific value
Jiri Borovec's avatar
Jiri Borovec committed
98
    SEE: https://en.wikipedia.org/wiki/Ellipse
Jiri Borovec's avatar
Jiri Borovec committed
99 100 101 102 103 104 105

    :param ratio: float, defining size of the ellipse to the image plane
    :param img: np.array<w, h>, while None, create empty one
    :param clr: int, value (0, 255) of an image intensity
    :param im_size: (int<w>, int<h>) image dimension
    :return: np.array<w, h>
    """
Jiri Borovec's avatar
Jiri Borovec committed
106
    logger.debug('draw an ellipse to an image with value %i', clr)
Jiri Borovec's avatar
Jiri Borovec committed
107 108
    center, radius = generate_rand_center_radius(img, ratio)
    x, y = draw.ellipse(center[0], center[1], radius[0], radius[1])
Jiri Borovec's avatar
Jiri Borovec committed
109
    img[x, y] = clr
Jiri Borovec's avatar
Jiri Borovec committed
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
    # img = transform.rotate(img, angle=random.randint(0, 180),
    #                        center=c, order=0, cval=img[0,0])
    # img = transform.rotate(img, angle=random.randint(0, 180),
    #                        center=np.array(im_size)/2, order=0, cval=img[0,0])
    return img


def draw_ellipsoid(img, ratio=0.1, clr=255):
    """ draw an ellipsoid to image plane with specific value
    SEE: https://en.wikipedia.org/wiki/Ellipsoid

    :param ratio: float, defining size of the ellipse to the image plane
    :param img: np.array<d, w, h>
    :param clr: int, value (0, 255) of an image intensity
    :return: np.array<d, w, h>
    """
    logger.debug('draw an ellipse to an image with value %i', clr)
    center, radius = generate_rand_center_radius(img, ratio)
    vec_dims = [np.arange(0, img.shape[i]) - center[i] for i in range(img.ndim)]
    Z, X, Y = np.meshgrid(*vec_dims, indexing='ij')
    a, b, c = radius
    dist = (Z ** 2 / a ** 2) + (X ** 2 / b ** 2) + (Y ** 2 / c ** 2)
    img[dist < 1.] = clr
Jiri Borovec's avatar
Jiri Borovec committed
133 134 135
    return img


Jiri Borovec's avatar
Jiri Borovec committed
136
def create_clean_folder(path_dir):
Jiri Borovec's avatar
Jiri Borovec committed
137 138
    """ create empty folder and while the folder exist clean all files

Jiri Borovec's avatar
Jiri Borovec committed
139
    :param path_dir: str, path
Jiri Borovec's avatar
Jiri Borovec committed
140 141
    :return: str
    """
Jiri Borovec's avatar
Jiri Borovec committed
142
    assert os.path.exists(os.path.dirname(path_dir)), os.path.dirname(path_dir)
Jiri Borovec's avatar
Jiri Borovec committed
143
    logger.info('create clean folder "%s"', path_dir)
Jiri Borovec's avatar
Jiri Borovec committed
144 145 146
    if os.path.exists(path_dir):
        shutil.rmtree(path_dir)
    os.mkdir(path_dir)
Jiri Borovec's avatar
Jiri Borovec committed
147
    return path_dir
Jiri Borovec's avatar
Jiri Borovec committed
148 149


150
def extract_image_largest_element(img_binary, labeled=None):
Jiri Borovec's avatar
Jiri Borovec committed
151 152 153
    """ take a binary image and find all independent segments,
    then keep just the largest segment and rest set as 0

154
    :param img_binary: np.array<w, h> of values {0, 1}
Jiri Borovec's avatar
Jiri Borovec committed
155 156
    :return: np.array<w, h> of values {0, 1}
    """
157 158
    if labeled is None or len(np.unique(labeled)) < 2:
        labeled, _ = ndimage.label(img_binary)
Jiri Borovec's avatar
Jiri Borovec committed
159
    areas = [(j, np.sum(labeled == j)) for j in np.unique(labeled)]
Jiri Borovec's avatar
Jiri Borovec committed
160
    areas = sorted(areas, key=lambda x: x[1], reverse=True)
161
    # logger.debug('... elements area: %s', repr(areas))
162
    img_ptn = img_binary.copy()
Jiri Borovec's avatar
Jiri Borovec committed
163
    if len(areas) > 1:
164
        img_ptn = np.zeros_like(img_binary)
Jiri Borovec's avatar
Jiri Borovec committed
165
        # skip largest, assuming to be background
166 167
        img_ptn[labeled == areas[1][0]] = 1
    return img_ptn
Jiri Borovec's avatar
Jiri Borovec committed
168 169


Jiri Borovec's avatar
Jiri Borovec committed
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
def atlas_filter_larges_components(atlas):
    # export to dictionary
    logger.info('... post-processing over generated patterns: %s',
                repr(np.unique(atlas).tolist()))
    atlas_new = np.zeros(atlas.shape, dtype=np.uint8)
    imgs_patterns = []
    for i, idx in enumerate(np.unique(atlas)[1:]):
        im = np.zeros(atlas.shape, dtype=np.uint8)
        # create pattern
        im[atlas == idx] = 1
        # remove all smaller unconnected elements
        im = extract_image_largest_element(im)
        if np.sum(im) == 0: continue
        imgs_patterns.append(im)
        # add them to the final arlas
        atlas_new[im == 1] = i + 1
    return atlas_new, imgs_patterns


def dictionary_generate_atlas(path_out, dir_name=DIR_NAME_DICTIONARY,
                              nb_ptns=NB_ATM_PATTERNS, im_size=IMAGE_SIZE_2D,
                              temp_img_name=IMAGE_PATTERN):
Jiri Borovec's avatar
Jiri Borovec committed
192 193
    """ generate pattern dictionary as atlas, no overlapping

Jiri Borovec's avatar
Jiri Borovec committed
194 195
    :param path_out: str, path to the results directory
    :param nb_ptns: int, number of patterns / labels
Jiri Borovec's avatar
Jiri Borovec committed
196 197 198
    :param im_size: (int<w>, int<h>)
    :return: [np.array<w, h>] list of independet patters in the dictionary
    """
Jiri Borovec's avatar
Jiri Borovec committed
199
    logger.info('generate an Atlas composed from %i patterns and image size %s',
Jiri Borovec's avatar
Jiri Borovec committed
200 201
                nb_ptns, repr(im_size))
    out_dir = os.path.join(path_out, dir_name)
Jiri Borovec's avatar
Jiri Borovec committed
202 203
    create_clean_folder(out_dir)
    atlas = np.zeros(im_size, dtype=np.uint8)
Jiri Borovec's avatar
Jiri Borovec committed
204
    for i in range(nb_ptns):
Jiri Borovec's avatar
Jiri Borovec committed
205 206 207 208 209
        label = (i + 1)
        if len(im_size) == 2:
            atlas = draw_ellipse(atlas, clr=label)
        elif len(im_size) == 3:
            atlas = draw_ellipsoid(atlas, clr=label)
Jiri Borovec's avatar
Jiri Borovec committed
210 211 212 213 214
    logger.debug(type(atlas))
    atlas = image_deform_elastic(atlas)
    logger.debug(np.unique(atlas))
    export_image(out_dir, atlas, 'atlas')
    # in case run in DEBUG show atlas and wait till close
Jiri Borovec's avatar
Jiri Borovec committed
215
    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
Jiri Borovec's avatar
Jiri Borovec committed
216
        logger.debug('labels: %s', repr(np.unique(atlas)))
Jiri Borovec's avatar
Jiri Borovec committed
217 218 219 220
        if atlas.ndim == 2:
            plt.imshow(atlas)
        else:
            plt.imshow(atlas[int(atlas.shape[0] / 2)])
Jiri Borovec's avatar
Jiri Borovec committed
221
        plt.show()
Jiri Borovec's avatar
Jiri Borovec committed
222
    atlas_new, imgs_patterns = atlas_filter_larges_components(atlas)
Jiri Borovec's avatar
Jiri Borovec committed
223
    export_image(out_dir, atlas_new, 'atlas')
Jiri Borovec's avatar
Jiri Borovec committed
224 225 226
    for i, img in enumerate(imgs_patterns):
        export_image(out_dir, img, i, temp_img_name)
    return imgs_patterns
Jiri Borovec's avatar
Jiri Borovec committed
227 228


Jiri Borovec's avatar
Jiri Borovec committed
229
def dictionary_generate_rnd_pattern(path_out, dir_name=DIR_NAME_DICTIONARY,
Jiri Borovec's avatar
Jiri Borovec committed
230
                                    nb_ptns=NB_ATM_PATTERNS, im_size=IMAGE_SIZE_2D,
Jiri Borovec's avatar
Jiri Borovec committed
231
                                    temp_img_name=IMAGE_PATTERN):
Jiri Borovec's avatar
Jiri Borovec committed
232 233
    """ generate pattern dictionary and allow overlapping

Jiri Borovec's avatar
Jiri Borovec committed
234 235
    :param path_out: str, path to the results directory
    :param nb_ptns: int, number of patterns / labels
Jiri Borovec's avatar
Jiri Borovec committed
236 237 238
    :param im_size: (int<w>, int<h>)
    :return: [np.array<w, h>] list of independent patters in the dictionary
    """
Jiri Borovec's avatar
Jiri Borovec committed
239
    logger.info('generate Dict. composed from %i patterns and img. size %s',
Jiri Borovec's avatar
Jiri Borovec committed
240 241
                nb_ptns, repr(im_size))
    out_dir = os.path.join(path_out, dir_name)
Jiri Borovec's avatar
Jiri Borovec committed
242
    create_clean_folder(out_dir)
Jiri Borovec's avatar
Jiri Borovec committed
243 244
    list_imgs = []
    for i in range(nb_ptns):
Jiri Borovec's avatar
Jiri Borovec committed
245
        im = draw_ellipse(np.zeros(im_size, dtype=np.uint8))
Jiri Borovec's avatar
Jiri Borovec committed
246
        im = image_deform_elastic(im)
Jiri Borovec's avatar
Jiri Borovec committed
247 248 249
        list_imgs.append(im)
        export_image(out_dir, im, i, temp_img_name)
    return list_imgs
Jiri Borovec's avatar
Jiri Borovec committed
250 251


Jiri Borovec's avatar
Jiri Borovec committed
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
def generate_rand_patterns_occlusion(idx, im_ptns, out_dir,
                                     ptn_ration=RND_PATTERN_OCCLUSION):
    """ generate the new sample from list of pattern with specific ration

    :param idx: int
    :param im_ptns: [np.arrays]
    :param out_dir: str
    :param ptn_ration: float in range (0, 1)
    :return: int, np.array, str, [int]
    """
    np.random.seed()  # reinit seed to have random samples even in the same time
    bool_combine = np.random.random(len(im_ptns)) < ptn_ration
    # if there is non above threshold select one random
    if not any(bool_combine):
        bool_combine[np.random.randint(0, len(bool_combine))] = True
    logger.debug('combination vector is %s', repr(bool_combine.tolist()))
    im = sum(np.asarray(im_ptns)[bool_combine])
    # convert sum to union such as all above 0 set as 1
    im[im > 0.] = 1
    im_name = SEGM_PATTERN.format(idx)
    export_image(out_dir, im, idx)
    ptn_weights = [int(x) for x in bool_combine]
    return idx, im, im_name, ptn_weights


Jiri Borovec's avatar
Jiri Borovec committed
277
def dataset_binary_combine_patterns(im_ptns, out_dir, nb_samples=NB_SAMPLES,
Jiri Borovec's avatar
Jiri Borovec committed
278 279
                                    ptn_ration=RND_PATTERN_OCCLUSION,
                                    nb_jobs=NB_THREADS):
Jiri Borovec's avatar
Jiri Borovec committed
280 281 282
    """ generate a Binary dataset composed from N samples and given ration
    of pattern occlusion

Jiri Borovec's avatar
Jiri Borovec committed
283
    :param nb_jobs: int
Jiri Borovec's avatar
Jiri Borovec committed
284
    :param im_ptns: [np.array<w, h>] list of ind. patters in the dictionary
Jiri Borovec's avatar
Jiri Borovec committed
285
    :param out_dir: str, path to the results directory
Jiri Borovec's avatar
Jiri Borovec committed
286
    :param nb_samples: int, number of samples in dataset
Jiri Borovec's avatar
Jiri Borovec committed
287 288 289 290
    :param ptn_ration: float, ration of how many patterns are used to create
        an input observation / image
    :return: [np.array<w, h>], df<nb_imgs, nb_lbs>
    """
Jiri Borovec's avatar
Jiri Borovec committed
291 292
    logger.info('generate a Binary dataset composed from %i samples  '
                'and ration pattern occlusion %f', nb_samples, ptn_ration)
Jiri Borovec's avatar
Jiri Borovec committed
293 294
    create_clean_folder(out_dir)
    df_weights = pd.DataFrame()
Jiri Borovec's avatar
Jiri Borovec committed
295 296 297
    im_spls = [None] * nb_samples
    mproc_pool = mproc.Pool(nb_jobs)
    logger.debug('running in %i threads...', nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
298
    tqdm_bar = tqdm.tqdm(total=nb_samples)
Jiri Borovec's avatar
Jiri Borovec committed
299 300 301 302 303 304 305
    for idx, im, im_name, ptn_weights in mproc_pool.imap_unordered(
            partial(generate_rand_patterns_occlusion, im_ptns=im_ptns,
                    out_dir=out_dir, ptn_ration=ptn_ration,), range(nb_samples)):
        im_spls[idx] = im
        df_weights = df_weights.append(pd.Series([im_name] + ptn_weights),
                                       ignore_index=True)

Jiri Borovec's avatar
Jiri Borovec committed
306
        tqdm_bar.update(1)
Jiri Borovec's avatar
Jiri Borovec committed
307 308
    mproc_pool.close()
    mproc_pool.join()
Jiri Borovec's avatar
Jiri Borovec committed
309 310
    df_weights.columns = ['image'] + [COLUMN_NAME.format(i + 1)
                                      for i in range(len(df_weights.columns) - 1)]
Jiri Borovec's avatar
Jiri Borovec committed
311
    df_weights.set_index('image', inplace=True)
Jiri Borovec's avatar
Jiri Borovec committed
312
    logger.debug(df_weights.head())
Jiri Borovec's avatar
Jiri Borovec committed
313 314 315 316
    return im_spls, df_weights


def add_image_binary_noise(im, ration=0.1):
Jiri Borovec's avatar
Jiri Borovec committed
317 318 319 320 321 322
    """ generate and add a binary noise to an image

    :param im: np.array<w, h> input binary image
    :param ration: float (0, 1) means 0 = no noise
    :return: np.array<w, h> binary image
    """
Jiri Borovec's avatar
Jiri Borovec committed
323 324 325 326 327 328 329 330 331 332 333
    logger.debug('... add random noise to a binary image')
    rnd = np.random.random(im.shape)
    rnd = np.array(rnd < ration, dtype=np.int16)
    im_noise = np.abs(np.asanyarray(im, dtype=np.int16) - rnd)
    # plt.subplot(1,3,1), plt.imshow(im)
    # plt.subplot(1,3,2), plt.imshow(rnd)
    # plt.subplot(1,3,3), plt.imshow(im - rnd)
    # plt.show()
    return np.array(im_noise, dtype=np.int16)


Jiri Borovec's avatar
Jiri Borovec committed
334
def export_image(path_out, img, im_name, name_template=SEGM_PATTERN):
Jiri Borovec's avatar
Jiri Borovec committed
335 336
    """ export an imahe with given path and optional pattern for image name

Jiri Borovec's avatar
Jiri Borovec committed
337 338
    :param path_out: str, path to the results directory
    :param img: np.array<w, h>
Jiri Borovec's avatar
Jiri Borovec committed
339
    :param im_name: str/int image nea of index to be place to patterns name
Jiri Borovec's avatar
Jiri Borovec committed
340
    :param name_template: str, while the name is not string generate image according
Jiri Borovec's avatar
Jiri Borovec committed
341 342 343
        specific pattern, like format fn
    :return: str, path to the image
    """
Jiri Borovec's avatar
Jiri Borovec committed
344 345 346
    if not os.path.exists(path_out):
        return ''
    if not isinstance(im_name, str):
Jiri Borovec's avatar
Jiri Borovec committed
347
        im_name = name_template.format(im_name)
Jiri Borovec's avatar
Jiri Borovec committed
348
    path_img = os.path.join(path_out, im_name)
Jiri Borovec's avatar
Jiri Borovec committed
349 350
    logger.debug(' .. saving image %s with %s to "%s...%s"', repr(img.shape),
                 repr(np.unique(img)), path_img[:25], path_img[-25:])
Jiri Borovec's avatar
Jiri Borovec committed
351
    if img.ndim == 2 or img.shape[2] <= 3:
Jiri Borovec's avatar
Jiri Borovec committed
352
        im_norm = img / float(np.max(img)) * 255
Jiri Borovec's avatar
Jiri Borovec committed
353
        # io.imsave(path_img, im_norm)
Jiri Borovec's avatar
Jiri Borovec committed
354
        Image.fromarray(im_norm.astype(np.uint8)).save(path_img + '.png')
Jiri Borovec's avatar
Jiri Borovec committed
355 356 357 358
    elif img.ndim == 3:
        img_clip = img / float(img.max()) * 255**2
        tif = libtiff.TIFF.open(path_img + '.tiff', mode='w')
        tif.write_image(img_clip.astype(np.uint16))
Jiri Borovec's avatar
Jiri Borovec committed
359
    return path_img
Jiri Borovec's avatar
Jiri Borovec committed
360 361


Jiri Borovec's avatar
Jiri Borovec committed
362 363 364 365 366 367 368 369
def wrapper_apply_function(i_img, func, coef, out_dir):
    i, img = i_img
    img_def = func(img, coef)
    export_image(out_dir, img_def, i)
    return i, img_def


def dataset_apply_image_function(imgs, out_dir, func, coef=0.5, nb_jobs=NB_THREADS):
Jiri Borovec's avatar
Jiri Borovec committed
370
    """ having list if input images create an dataset with randomly deform set
Jiri Borovec's avatar
Jiri Borovec committed
371
    of these images and export them to the results folder
Jiri Borovec's avatar
Jiri Borovec committed
372

Jiri Borovec's avatar
Jiri Borovec committed
373 374
    :param nb_jobs:
    :param func:
Jiri Borovec's avatar
Jiri Borovec committed
375
    :param imgs: [np.array<w, h>] raw input images
Jiri Borovec's avatar
Jiri Borovec committed
376
    :param out_dir: str, path to the results directory
Jiri Borovec's avatar
Jiri Borovec committed
377 378 379
    :param coef: float, a param describing the how much it is deformed (0 = None)
    :return: [np.array<w, h>]
    """
Jiri Borovec's avatar
Jiri Borovec committed
380 381
    logger.info('apply costume funstion "%s" on %i samples with coef. %f',
                func.__name__, len(imgs), coef)
Jiri Borovec's avatar
Jiri Borovec committed
382 383
    create_clean_folder(out_dir)

Jiri Borovec's avatar
Jiri Borovec committed
384 385 386 387 388 389 390 391 392 393 394
    imgs_new = [None] * len(imgs)
    mproc_pool = mproc.Pool(nb_jobs)
    logger.debug('running in %i threads...', nb_jobs)
    tqdm_bar = tqdm.tqdm(total=len(imgs))
    for i, im in mproc_pool.imap_unordered(partial(wrapper_apply_function,
                                        func=func, coef=coef, out_dir=out_dir),
                                enumerate(imgs)):
        imgs_new[i] = im
        tqdm_bar.update(1)
    mproc_pool.close()
    mproc_pool.join()
Jiri Borovec's avatar
Jiri Borovec committed
395

Jiri Borovec's avatar
Jiri Borovec committed
396
    return imgs_new
Jiri Borovec's avatar
Jiri Borovec committed
397 398 399


def image_transform_binary2prob(im, coef=0.1):
Jiri Borovec's avatar
Jiri Borovec committed
400 401 402 403 404 405 406
    """ convert a binary image to probability while computing distance function
    on the binary function (contours)

    :param im: np.array<w, h> input binary image
    :param coef: float, influence hoe strict the boundary between F-B is
    :return: np.array<w, h> float image
    """
Jiri Borovec's avatar
Jiri Borovec committed
407
    logger.debug('... transform binary image to probability')
Jiri Borovec's avatar
Jiri Borovec committed
408 409 410 411 412 413 414 415 416 417 418
    im_dist = ndimage.distance_transform_edt(im)
    im_dist -= ndimage.distance_transform_edt(1-im)
    im_prob = 1. / (1. + np.exp(-coef * im_dist))
    # plt.subplot(1,3,1), plt.imshow(im)
    # plt.subplot(1,3,2), plt.imshow(im_dist)
    # plt.subplot(1,3,3), plt.imshow(im_prob)
    # plt.show()
    return im_prob


def add_image_prob_noise(im, ration=0.1):
Jiri Borovec's avatar
Jiri Borovec committed
419 420 421 422 423 424
    """ generate and add a continues noise to an image

    :param im: np.array<w, h> input float image
    :param ration: float (0, 1) means 0 = no noise
    :return: np.array<w, h> float image
    """
Jiri Borovec's avatar
Jiri Borovec committed
425
    logger.debug('... add smooth noise to a probability image')
Jiri Borovec's avatar
Jiri Borovec committed
426
    rnd = 2 * (np.random.random(im.shape) - 0.5)
Jiri Borovec's avatar
Jiri Borovec committed
427 428 429 430 431 432 433 434 435
    rnd[abs(rnd) > ration] = 0
    im_noise = np.abs(im - rnd)
    # plt.subplot(1,3,1), plt.imshow(im)
    # plt.subplot(1,3,2), plt.imshow(rnd)
    # plt.subplot(1,3,3), plt.imshow(im - rnd)
    # plt.show()
    return im_noise


436
def wrapper_load_images(list_path_img):
Jiri Borovec's avatar
Jiri Borovec committed
437 438 439 440 441
    logger.debug('parallel loading %i images', len(list_path_img))
    list_names_imgs = map(load_image, list_path_img)
    return list_names_imgs


Jiri Borovec's avatar
Jiri Borovec committed
442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
def find_images(path_dir, im_pattern='*', img_posixs=IMAGE_POSIXS):
    """ in given folder find largest group of equal images types

    :param path_dir: str
    :param im_pattern: str
    :param img_posixs: [str]
    :return: [str]
    """
    paths_img_most = []
    for im_posix in img_posixs:
        path_search = os.path.join(path_dir, im_pattern + im_posix)
        paths_img = glob.glob(path_search)
        logger.debug('images found %i for search "%s"', len(paths_img), path_search)
        if len(paths_img) > len(paths_img_most):
            paths_img_most = paths_img
    return paths_img_most


def dataset_load_images(path_dir, im_pattern='*', nb_spls=None, nb_jobs=1):
Jiri Borovec's avatar
Jiri Borovec committed
461 462 463
    """ load complete dataset or just a subset

    :param name: str, name od particular dataset
Jiri Borovec's avatar
Jiri Borovec committed
464
    :param path_base: str, path to the results directory
Jiri Borovec's avatar
Jiri Borovec committed
465
    :param im_pattern: str, specific pattern of loaded images
Jiri Borovec's avatar
Jiri Borovec committed
466 467
    :param im_posix: str image pattern line '.png'
    :param nb_spls: int, number of samples to be loaded, None means all
Jiri Borovec's avatar
Jiri Borovec committed
468 469
    :param nb_jobs: int
    :return: [np.array], [str]
Jiri Borovec's avatar
Jiri Borovec committed
470
    """
Jiri Borovec's avatar
Jiri Borovec committed
471
    logger.debug('loading folder (%s) <- "%s"', os.path.exists(path_dir), path_dir)
Jiri Borovec's avatar
Jiri Borovec committed
472
    assert os.path.exists(path_dir), path_dir
Jiri Borovec's avatar
Jiri Borovec committed
473 474
    paths_img = find_images(path_dir, im_pattern)
    paths_img = sorted(paths_img)[:nb_spls]
Jiri Borovec's avatar
Jiri Borovec committed
475
    logger.debug('number samples %i in dataset "%s"', len(paths_img),
Jiri Borovec's avatar
Jiri Borovec committed
476
                 os.path.basename(path_dir))
477 478

    if nb_jobs > 1:
Jiri Borovec's avatar
Jiri Borovec committed
479
        logger.debug('running in %i threads...', nb_jobs)
480
        nb_load_blocks = len(paths_img) / BLOCK_NB_LOAD_IMAGES
Jiri Borovec's avatar
Jiri Borovec committed
481
        logger.debug('estimated %i loading blocks', nb_load_blocks)
482
        block_paths_img = (paths_img[i::nb_load_blocks] for i in range(nb_load_blocks))
Jiri Borovec's avatar
Jiri Borovec committed
483

484
        mproc_pool = mproc.Pool(nb_jobs)
485
        list_names_imgs = mproc_pool.map(wrapper_load_images, block_paths_img)
486 487
        mproc_pool.close()
        mproc_pool.join()
488 489 490
        logger.debug('transforming the parallel results')
        names_imgs = sorted(itertools.chain(*list_names_imgs))
        im_names, imgs = zip(*names_imgs)
491
    else:
Jiri Borovec's avatar
Jiri Borovec committed
492
        logger.debug('running in single thread...')
493 494
        names_imgs = [load_image(p) for p in paths_img]
        logger.debug('split the resulting tuples')
Jiri Borovec's avatar
Jiri Borovec committed
495
        im_names, imgs = zip(*names_imgs)
496
    assert len(paths_img) == len(imgs)
Jiri Borovec's avatar
Jiri Borovec committed
497
    return imgs, im_names
Jiri Borovec's avatar
Jiri Borovec committed
498 499


Jiri Borovec's avatar
Jiri Borovec committed
500
def load_image(path_img):
Jiri Borovec's avatar
Jiri Borovec committed
501
    assert os.path.exists(path_img), path_img
Jiri Borovec's avatar
Jiri Borovec committed
502
    n_img, img_ext = os.path.splitext(os.path.basename(path_img))
Jiri Borovec's avatar
Jiri Borovec committed
503
    if img_ext in ['.tif', '.tiff']:
Jiri Borovec's avatar
Jiri Borovec committed
504 505 506 507 508 509 510 511
        im = libtiff.TiffFile(path_img).get_tiff_array()
        img = np.empty(im.shape)
        for i in range(img.shape[0]):
            img[i, :, :] = im[i]
        img = np.array(img.tolist())
    else:
        # img = io.imread(path_img)
        img = np.array(Image.open(path_img))
Jiri Borovec's avatar
Jiri Borovec committed
512
    img = (img / float(img.max()))
Jiri Borovec's avatar
Jiri Borovec committed
513
    return n_img, img
514 515


Jiri Borovec's avatar
Jiri Borovec committed
516
def dataset_load_weights(path_base, name_csv=CSV_NAME_WEIGHTS):
Jiri Borovec's avatar
Jiri Borovec committed
517 518
    """ loading all true wieghts for given dataset

Jiri Borovec's avatar
Jiri Borovec committed
519
    :param path_base: str, path to the results directory
Jiri Borovec's avatar
Jiri Borovec committed
520
    :param name_csv: str, name of file with weights
Jiri Borovec's avatar
Jiri Borovec committed
521 522
    :return: np.array<nb_imgs, nb_lbs>
    """
Jiri Borovec's avatar
Jiri Borovec committed
523 524 525 526 527 528 529 530 531 532
    path_csv = os.path.join(path_base, name_csv)
    df = pd.DataFrame().from_csv(path_csv)
    # for the original encoding as string in single column
    if 'combination' in df.columns:
        coding = df['combination'].values.tolist()
        logger.debug('encoding of length: %i', len(coding))
        encoding = np.array([[int(x) for x in c.split(';')] for c in coding])
    # the new encoding with pattern names
    else:
        encoding = df.as_matrix()
Jiri Borovec's avatar
Jiri Borovec committed
533 534 535
    return np.array(encoding)


Jiri Borovec's avatar
Jiri Borovec committed
536
def dataset_compose_atlas(path_base, name=DIR_NAME_DICTIONARY, img_temp_name='pattern_*'):
Jiri Borovec's avatar
Jiri Borovec committed
537 538 539
    """ load all independent patterns and compose them into single m-label atlas

    :param name: str, name of dataset
Jiri Borovec's avatar
Jiri Borovec committed
540
    :param path_base: str, path to the results directory
Jiri Borovec's avatar
Jiri Borovec committed
541
    :param img_temp_name: str
Jiri Borovec's avatar
Jiri Borovec committed
542 543
    :return: np.array<w, h>
    """
Jiri Borovec's avatar
Jiri Borovec committed
544
    imgs, _ = dataset_load_images(os.path.join(path_base, name), img_temp_name)
Jiri Borovec's avatar
Jiri Borovec committed
545 546 547 548 549 550 551
    assert len(imgs) > 0
    atlas = np.zeros_like(imgs[0])
    for i, im in enumerate(imgs):
        atlas[ im == 1 ] = i+1
    return np.array(atlas, dtype=np.uint8)


552
def dataset_export_images(p_out, imgs, names=None, nb_jobs=1):
Jiri Borovec's avatar
Jiri Borovec committed
553 554
    """ export complete dataset

555
    :param parallel: bool
Jiri Borovec's avatar
Jiri Borovec committed
556 557 558 559 560
    :param p_out: str
    :param imgs: [np.array<w, h>]
    :param names: [str] or None (use indexes)
    """
    create_clean_folder(p_out)
Jiri Borovec's avatar
Jiri Borovec committed
561
    logger.debug('export %i images into "%s"', len(imgs), p_out)
Jiri Borovec's avatar
Jiri Borovec committed
562 563
    if names is None:
        names = range(len(imgs))
564

Jiri Borovec's avatar
Jiri Borovec committed
565
    mp_set = [(p_out, im, names[i]) for i, im in enumerate(sorted(imgs))]
566
    if nb_jobs > 1:
Jiri Borovec's avatar
Jiri Borovec committed
567
        logger.debug('running in %i threads...', nb_jobs)
568
        mproc_pool = mproc.Pool(nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
569
        mproc_pool.map(wrapper_export_image, mp_set)
570 571 572
        mproc_pool.close()
        mproc_pool.join()
    else:
Jiri Borovec's avatar
Jiri Borovec committed
573
        logger.debug('running in single thread...')
Jiri Borovec's avatar
Jiri Borovec committed
574
        map(wrapper_export_image, mp_set)
Jiri Borovec's avatar
Jiri Borovec committed
575 576 577 578 579 580
    # try:
    #     path_npz = os.path.join(p_out, 'input_images.npz')
    #     np.savez(open(path_npz, 'w'), imgs)
    # except:
    #     logger.error(traceback.format_exc())
    #     os.remove(path_npz)
Jiri Borovec's avatar
Jiri Borovec committed
581 582


Jiri Borovec's avatar
Jiri Borovec committed
583
def wrapper_export_image(mp_set):
Jiri Borovec's avatar
Jiri Borovec committed
584
    export_image(*mp_set)
585 586


Jiri Borovec's avatar
Jiri Borovec committed
587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
# def dataset_convert_nifti(path_in, path_out, img_posix=IMAGE_POSIX):
#     """ having a datset of png images conver them into nifti images
#
#     :param path_in: str
#     :param path_out: str
#     :param img_posix: str, like '.png'
#     :return:
#     """
#     import src.own_utils.tool_data_io as tl_data
#     logger.info('convert a dataset to Nifti')
#     p_imgs = glob.glob(os.path.join(path_in, '*' + img_posix))
#     create_clean_folder(path_out)
#     p_imgs = sorted(p_imgs)
#     for path_im in p_imgs:
#         name = os.path.splitext(os.path.basename(path_im))[0]
#         path_out = os.path.join(path_out, name)
#         logger.debug('... converting "%s" -> "%s"', path_im, path_out)
#         tl_data.convert_img_2_nifti_gray(path_im, path_out)
#     return None
Jiri Borovec's avatar
Jiri Borovec committed
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626


def get_simple_atlas():
    atlas = np.zeros((20,20))
    atlas[2:8,2:8] = 1
    atlas[12:18,12:18] = 2
    atlas[2:8,12:18] = 3
    return atlas


def get_sample_images(atlas):
    im1 = atlas.copy()
    im1[im1>=2] = 0
    im2 = atlas.copy()
    im2[im2<=1] = 0
    im2[im2>0] = 1
    im3 = atlas.copy()
    im3[atlas<2] = 0
    im3[atlas>2] = 0
    im3[im3>0] = 1
    return im1, im2, im3