run_apd_reconstruction.py 16.1 KB
Newer Older
Jiri Borovec's avatar
Jiri Borovec committed
1 2 3
"""
script that take th csv files with encoding with the proposed atlas
and does the back reconstruction of each image. As sub-step it compute
Jiri Borovec's avatar
Jiri Borovec committed
4
the reconstruction error to evaluate he parameters and export visualisation
Jiri Borovec's avatar
Jiri Borovec committed
5

Jiri Borovec's avatar
Jiri Borovec committed
6 7 8
EXAMPLE:
>> python run_apd_reconstruction.py \
    --path_in /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APDL_real
9 10 11 12 13 14 15

>> python run_apd_reconstruction.py \
    --path_in /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_real

>> python run_apd_reconstruction.py \
    --path_in /datagrid/Medical/microscopy/drosophila/RESULTS/experiments_APD_real

Jiri Borovec's avatar
Jiri Borovec committed
16 17 18 19 20
>> python run_apd_reconstruction.py \
    --path_in /datagrid/Medical/microscopy/drosophila/TEMPORARY/experiments_APD_temp \
    --names_expt ExperimentALPE_mp_real_type_3_segm_reg_binary_gene_ssmall_20160509-155333 \
    --nb_jobs 1

Jiri Borovec's avatar
Jiri Borovec committed
21
Copyright (C) 2015-2016 Jiri Borovec <jiri.borovec@fel.cvut.cz>
Jiri Borovec's avatar
Jiri Borovec committed
22 23 24 25 26 27
"""

import os
import glob
import json
import logging
Jiri Borovec's avatar
Jiri Borovec committed
28 29
import time
import gc
Jiri Borovec's avatar
Jiri Borovec committed
30
import argparse
Jiri Borovec's avatar
Jiri Borovec committed
31
import multiprocessing as mproc
Jiri Borovec's avatar
Jiri Borovec committed
32
from functools import partial
Jiri Borovec's avatar
Jiri Borovec committed
33

Jiri Borovec's avatar
Jiri Borovec committed
34
import tqdm
Jiri Borovec's avatar
Jiri Borovec committed
35 36 37 38 39
import numpy as np
import pandas as pd
from PIL import Image
import matplotlib.pyplot as plt

Jiri Borovec's avatar
Jiri Borovec committed
40
import pattern_weights as ptn_weight
Jiri Borovec's avatar
Jiri Borovec committed
41
import dataset_utils as gen_data
Jiri Borovec's avatar
Jiri Borovec committed
42

43
VISUAL = True
Jiri Borovec's avatar
Jiri Borovec committed
44
NB_THREADS = int(mproc.cpu_count() * .9)
45
PATH_EXPERIMENTS = '/datagrid/Medical/microscopy/drosophila/RESULTS/' \
46
                   'experiments_APD_real'
47 48 49
# PATH_EXPERIMENTS = '/datagrid/Medical/microscopy/drosophila/TEMPORARY/' \
#                    'experiments_APD_real'
PATH_IMAGES_RGB = '/datagrid/Medical/microscopy/drosophila/TEMPORARY'
Jiri Borovec's avatar
Jiri Borovec committed
50

Jiri Borovec's avatar
Jiri Borovec committed
51 52 53 54 55
DICT_PARAMS = {
    'path_in': PATH_EXPERIMENTS,
    'names_expt': None,
    'nb_jobs': NB_THREADS,
}
Jiri Borovec's avatar
Jiri Borovec committed
56

Jiri Borovec's avatar
Jiri Borovec committed
57 58 59 60
NAME_CONFIG = 'config.json'
PREFIX_ATLAS = 'atlas_'
PREFIX_ENCODE = 'encoding_'
PREFIX_RECONST = 'reconstruct_'
Jiri Borovec's avatar
Jiri Borovec committed
61 62
CSV_RECONT_DIFF = 'reconstruction_diff.csv'
TXT_RECONT_DIFF = 'reconstruction_diff.txt'
Jiri Borovec's avatar
Jiri Borovec committed
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
POSIX_CSV_NEW = '_geneNew.csv'


def create_args_parser(params=DICT_PARAMS):
    """ create simple arg parser with default values (input, output, dataset)

    :param params: {str: ...}
    :return: object argparse<in, ...>
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-in', '--path_in', type=str, required=False,
                        default=params['path_in'],
                        help='path to the input image dataset')
    parser.add_argument('-expt', '--names_expt', type=str, required=False, nargs='+',
                        default=params['names_expt'],
                        help='name of the experiment')
    parser.add_argument('--nb_jobs', type=int, required=False,
                        default=params['nb_jobs'],
                        help='number of processes running in parallel')
    return parser


def parse_arg_params(parser):
    """ parse basic args and return as dictionary

    :param parser: argparse
    :return: {str: ...}
    """
    args = vars(parser.parse_args())
    args['path_in'] = os.path.abspath(os.path.expanduser(args['path_in']))
    assert os.path.exists(args['path_in']), '%s' % args['path_in']
    if args['names_expt'] is None:
        args['names_expt'] = sorted([os.path.basename(p) for p
                                    in glob.glob(os.path.join(args['path_in'], '*'))
                                    if os.path.isdir(p)])
    elif isinstance(args['names_expt'], str):
        args['names_expt'] = [args['names_expt']]  # make it as list with single
    args['names_expt'] = [n for n in args['names_expt']
                         if os.path.exists(os.path.join(args['path_in'], n))]
    return args
Jiri Borovec's avatar
Jiri Borovec committed
103 104


105 106
def export_fig_reconstruction(path_out, name, segm_orig, segm_reconst, img_atlas,
                              img_rgb=None):
Jiri Borovec's avatar
Jiri Borovec committed
107 108 109 110 111
    """ visualise reconstruction together with the original segmentation

    :param path_out: str
    :param name: str
    :param segm_orig: np.array<height, width>
Jiri Borovec's avatar
Jiri Borovec committed
112 113
    :param segm_reconst: np.array<height, width>
    :param img_atlas: np.array<height, width>
Jiri Borovec's avatar
Jiri Borovec committed
114
    """
Jiri Borovec's avatar
Jiri Borovec committed
115
    fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(20, 8))
116 117 118 119 120 121
    ax[0].set_title('original')
    if img_rgb is not None:
        ax[0].imshow(img_rgb, alpha=0.9)
    else:
        ax[0].imshow(1 - segm_orig, cmap='Greys', alpha=0.7)
    ax[0].imshow(img_atlas, alpha=0.1)
122
    ax[0].axis('off')
123 124 125
    ax[0].contour(img_atlas > 0, linewidth=2)
    ax[0].contour(img_atlas, levels=np.unique(img_atlas),
                  linewidth=2, cmap=plt.cm.jet)
126

Jiri Borovec's avatar
Jiri Borovec committed
127
    ax[1].set_title('reconstructed segmentation')
128 129 130 131 132 133 134 135
    lut = plt.cm.get_cmap('jet', segm_reconst.max())(range(segm_reconst.max() + 1))
    im = lut[segm_reconst]
    im[segm_reconst == 0, :] = (1., 1., 1., 1.)
    ax[1].imshow(im, alpha=0.8)
    ax[1].axes.get_xaxis().set_ticklabels([])
    ax[1].axes.get_yaxis().set_ticklabels([])
    # ax[1].axis('off')
    ax[1].contour(segm_orig, levels=np.unique(segm_orig), linewidth=2, colors='k')
136

Jiri Borovec's avatar
Jiri Borovec committed
137
    ax[2].set_title('selected vs not selected segm.')
138 139 140 141 142 143 144 145 146 147 148 149 150
    # segm_select = np.array(segm_reconst > 0, dtype=np.int)
    # segm_select[np.logical_and(img_atlas > 0, segm_reconst == 0)] = -1
    # segm_select[0, :2] = [-1, 1]
    # ax[2].imshow(segm_select, cmap=plt.cm.RdYlGn)
    im = np.ones(segm_reconst.shape + (4,))
    im[np.logical_and(img_atlas > 0, segm_reconst > 0), :] = (0, 1, 0, 1)
    im[np.logical_and(img_atlas > 0, segm_reconst == 0), :] = (1, 0, 0, 1)
    ax[2].imshow(im, alpha=0.7)
    ax[2].axes.get_xaxis().set_ticklabels([])
    ax[2].axes.get_yaxis().set_ticklabels([])
    # ax[2].axis('off')
    ax[2].contour(img_atlas, levels=np.unique(img_atlas), linewidth=1, colors='w')
    ax[2].contour(segm_orig, levels=np.unique(segm_orig), linewidth=3, colors='k')
151

Jiri Borovec's avatar
Jiri Borovec committed
152 153 154 155 156
    p_fig = os.path.join(path_out, name + '.png')
    fig.savefig(p_fig, bbox_inches='tight')
    plt.close(fig)


Jiri Borovec's avatar
Jiri Borovec committed
157 158
def compute_reconstruction(img_name, dict_params, path_out, im_atlas,
                           weights=None, b_visu=False):
Jiri Borovec's avatar
Jiri Borovec committed
159 160 161
    """ reconstruct the segmentation from atlas and particular weights
    and compute the reconstruction error

Jiri Borovec's avatar
Jiri Borovec committed
162
    :param path_out: str
Jiri Borovec's avatar
Jiri Borovec committed
163
    :param dict_params:
Jiri Borovec's avatar
Jiri Borovec committed
164
    :param im_atlas: np.array<height, width>
Jiri Borovec's avatar
Jiri Borovec committed
165 166
    :param weights: [<0,1>]
    :param img_name: str
Jiri Borovec's avatar
Jiri Borovec committed
167
    :param b_visu: bool
Jiri Borovec's avatar
Jiri Borovec committed
168 169 170
    :return: {str: float}
    """
    segm_orig = load_segmentation(dict_params, img_name)
171
    img_rgb = load_image_rgb(dict_params, img_name)
Jiri Borovec's avatar
Jiri Borovec committed
172 173 174 175
    if weights is None:
        # recompute encoding and then does the reconstruction
        weights = ptn_weight.weights_image_atlas_overlap_major(segm_orig, im_atlas)
    segm_rect = np.zeros_like(im_atlas)
Jiri Borovec's avatar
Jiri Borovec committed
176
    for i, w in enumerate(weights):
Jiri Borovec's avatar
Jiri Borovec committed
177
        lb = i + 1
Jiri Borovec's avatar
Jiri Borovec committed
178
        if w == 1:
Jiri Borovec's avatar
Jiri Borovec committed
179 180 181
            segm_rect[im_atlas == lb] = lb
    logging.debug('segm unique: %s', repr(np.unique(segm_rect)))
    if b_visu:
Jiri Borovec's avatar
Jiri Borovec committed
182
        try:
183 184
            export_fig_reconstruction(path_out, img_name, segm_orig, segm_rect,
                                      im_atlas)
Jiri Borovec's avatar
Jiri Borovec committed
185
        except:
Jiri Borovec's avatar
Jiri Borovec committed
186 187
            # logging.error(traceback.format_exc())
            logging.warning('drawing fail for "%s"...', img_name)
Jiri Borovec's avatar
Jiri Borovec committed
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
    segm_bin = (segm_rect >= 1)
    diff = np.sum(segm_orig != segm_bin) / float(np.prod(segm_orig.shape))
    return img_name, diff


def find_relevant_atlas(name_csv, list_names_atlas):
    """ find match of encode among all possible atlases

    :param name_csv: str
    :param list_names_atlas: [str]
    :return: str
    """
    list_atlas = [n.replace(PREFIX_ATLAS, '').replace('.png', '')
                  for n in list_names_atlas]
    name = name_csv.replace(PREFIX_ENCODE, '').replace('.csv', '')
    if name in list_atlas:
        idx = list_atlas.index(name)
        return list_names_atlas[idx]


Jiri Borovec's avatar
Jiri Borovec committed
208
def load_atlas_image(path_atlas):
Jiri Borovec's avatar
Jiri Borovec committed
209 210
    """ load the atlas as norm labels to be small natural ints

Jiri Borovec's avatar
Jiri Borovec committed
211
    :param path_atlas: str
Jiri Borovec's avatar
Jiri Borovec committed
212 213
    :return: np.array<height, width>
    """
Jiri Borovec's avatar
Jiri Borovec committed
214 215
    assert os.path.exists(path_atlas)
    img_atlas = Image.open(path_atlas)
Jiri Borovec's avatar
Jiri Borovec committed
216
    # norm image to have labels as [0, 1, 2, ...]
Jiri Borovec's avatar
Jiri Borovec committed
217 218 219 220 221
    uq_labels = sorted([lb for nb, lb in img_atlas.getcolors() if nb > 0])
    lut = np.zeros(max(uq_labels) + 1)
    for i, lb in enumerate(uq_labels):
        lut[lb] = i
    img_atlas = lut[np.array(img_atlas)]
Jiri Borovec's avatar
Jiri Borovec committed
222 223
    # subtract background (set it as -1)
    # img_atlas -= 1
Jiri Borovec's avatar
Jiri Borovec committed
224 225
    logging.debug('Atlas: %s with labels: %s', repr(img_atlas.shape),
                  repr(np.unique(img_atlas).tolist()))
Jiri Borovec's avatar
Jiri Borovec committed
226
    return img_atlas.astype(np.int)
Jiri Borovec's avatar
Jiri Borovec committed
227 228 229 230 231 232 233 234 235 236


def load_segmentation(dict_params, img_name):
    """ load the segmenattion with values {0, 1}

    :param dict_params: {str: values}
    :param img_name: str
    :return: np.array<height, width>
    """
    path_img = os.path.join(dict_params['path_in'], dict_params['dataset'],
Jiri Borovec's avatar
Jiri Borovec committed
237
                            img_name + '.png')
Jiri Borovec's avatar
Jiri Borovec committed
238 239 240 241 242
    img = np.array(Image.open(path_img))
    img /= img.max()
    return img


243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
def load_image_rgb(dict_params, img_name):
    """ load the segmenattion with values {0, 1}

    :param dict_params: {str: values}
    :param img_name: str
    :return: np.array<height, width>
    """
    path_img = os.path.join(dict_params['path_rgb'], img_name + '.png')
    if img_name == 'mean':
        return None
    if os.path.exists(path_img):
        img = np.array(Image.open(path_img))
    else:
        logging.warning('particular RGB image not exists "%s"', path_img)
        img = None
    return img


Jiri Borovec's avatar
Jiri Borovec committed
261 262
def export_fig_atlas(img_atlas, path_out, name, max_label=None):
    """ export the atlas to given folder and specific name
Jiri Borovec's avatar
Jiri Borovec committed
263

Jiri Borovec's avatar
Jiri Borovec committed
264 265 266 267 268 269
    :param path_out: str
    :param name: str
    :param img_atlas: np.array<height, width>
    """
    if max_label is None:
        max_label = np.max(img_atlas)
Jiri Borovec's avatar
Jiri Borovec committed
270
    fig = plt.figure(figsize=(8, 6))
Jiri Borovec's avatar
Jiri Borovec committed
271 272
    ax = fig.gca().imshow(img_atlas, vmax=max_label, interpolation='nearest')
    boundaries = (np.array(range(max_label + 1)) - 0.5).tolist() + [max_label + 0.5]
Jiri Borovec's avatar
Jiri Borovec committed
273
    cbar = plt.colorbar(ax, boundaries=boundaries)
Jiri Borovec's avatar
Jiri Borovec committed
274 275
    cbar.set_ticks(range(max_label + 1))
    cbar.set_ticklabels(['ptn %i' % lb for lb in range(max_label + 1)])
Jiri Borovec's avatar
Jiri Borovec committed
276 277 278 279
    fig.savefig(os.path.join(path_out, name + '_rgb.png'), bbox_inches='tight')
    plt.close(fig)


280
def perform_reconstruction_mproc(dict_params, name_csv, name_atlas, df_encode, img_atlas,
Jiri Borovec's avatar
Jiri Borovec committed
281 282 283 284 285 286 287 288 289 290 291
                                 nb_jobs=NB_THREADS, b_visu=False):
    """ perform the reconstruction in multi process mode

    :param dict_params: {}
    :param name_csv: str
    :param df_encode: DF
    :param img_atlas: np.array<height, width>
    :param nb_jobs: int
    :param b_visu: bool
    :return: DF
    """
292
    export_fig_atlas(img_atlas, dict_params['path_exp'], name_atlas)
Jiri Borovec's avatar
Jiri Borovec committed
293 294
    path_out = os.path.join(dict_params['path_exp'],
                            name_csv.replace(PREFIX_ENCODE, PREFIX_RECONST))
Jiri Borovec's avatar
Jiri Borovec committed
295
    if b_visu:
Jiri Borovec's avatar
Jiri Borovec committed
296 297
        if not os.path.exists(path_out):
            os.mkdir(path_out)
Jiri Borovec's avatar
Jiri Borovec committed
298
        export_fig_atlas(img_atlas, path_out, 'atlas')
Jiri Borovec's avatar
Jiri Borovec committed
299
    list_patterns = [col for col in df_encode.columns if col.startswith('ptn')]
Jiri Borovec's avatar
Jiri Borovec committed
300
    logging.debug('list of pattern names: %s', repr(list_patterns))
Jiri Borovec's avatar
Jiri Borovec committed
301 302
    list_idxs = [int(col[3:]) for col in list_patterns]
    assert list_idxs == sorted(list_idxs)
Jiri Borovec's avatar
Jiri Borovec committed
303
    assert np.max(img_atlas) <= len(list_patterns)
Jiri Borovec's avatar
Jiri Borovec committed
304

Jiri Borovec's avatar
Jiri Borovec committed
305
    wrapper_reconstruction = partial(compute_reconstruction, dict_params=dict_params,
306 307 308
                                     path_out=path_out, im_atlas=img_atlas, b_visu=b_visu)
    results = []
    tqdm_bar = tqdm.tqdm(total=len(df_encode))
Jiri Borovec's avatar
Jiri Borovec committed
309

Jiri Borovec's avatar
Jiri Borovec committed
310
    if nb_jobs > 1:
Jiri Borovec's avatar
Jiri Borovec committed
311
        logging.debug('computing %i samples in %i threads', len(df_encode), nb_jobs)
Jiri Borovec's avatar
Jiri Borovec committed
312
        mproc_pool = mproc.Pool(nb_jobs)
313 314 315
        for res in mproc_pool.imap_unordered(wrapper_reconstruction, df_encode.index):
            results.append(res)
            tqdm_bar.update()
Jiri Borovec's avatar
Jiri Borovec committed
316 317 318
        mproc_pool.close()
        mproc_pool.join()
    else:
319 320 321
        for res in map(wrapper_reconstruction, df_encode.index):
            results.append(res)
            tqdm_bar.update()
Jiri Borovec's avatar
Jiri Borovec committed
322 323

    df_diff = pd.DataFrame(results, columns=['image', name_csv])
Jiri Borovec's avatar
Jiri Borovec committed
324 325
    df_diff.set_index('image', inplace=True)
    logging.debug(repr(df_diff.describe()))
Jiri Borovec's avatar
Jiri Borovec committed
326 327 328
    return df_diff


Jiri Borovec's avatar
Jiri Borovec committed
329
def load_config_json(path_expt):
Jiri Borovec's avatar
Jiri Borovec committed
330 331 332 333 334 335
    """ load configuration from a experiment folder and correct
    some old parameter names to be compatible between ald and new versions

    :param path_expt: str
    :return: {str: ...}
    """
Jiri Borovec's avatar
Jiri Borovec committed
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
    with open(os.path.join(path_expt, NAME_CONFIG), 'r') as fp:
        dict_params = json.load(fp)
    # in case it was moved somewhere else
    dict_params['path_exp'] = path_expt
    for n in ['out_path', 'exp_path']:
        if n in dict_params:
            dict_params.pop(n)
    dict_params['path_out'] = os.path.dirname(path_expt)
    if 'in_path' in dict_params:
        dict_params['path_in'] = dict_params.pop('in_path')
    if 'sub_dataset' in dict_params:
        dict_params['path_in'] = os.path.join(dict_params['path_in'],
                                              dict_params.pop('dataset'))
        dict_params['dataset'] = dict_params.pop('sub_dataset')
    with open(os.path.join(path_expt, NAME_CONFIG), 'w') as fp:
        json.dump(dict_params, fp)
    return dict_params


Jiri Borovec's avatar
Jiri Borovec committed
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
def recompute_encoding(config, atlas):
    """ load images and according given atlas recompute the encoding

    :param config: {str: ...}
    :param atlas: np.array<height, width>
    :return: DF
    """
    path_in = os.path.join(config.get('path_in'), config.get('dataset'))
    imgs, im_names = gen_data.dataset_load_images(path_in)
    weights = [ptn_weight.weights_image_atlas_overlap_major(img, atlas) for img in imgs]
    df = pd.DataFrame(data=np.array(weights), index=im_names)
    df.columns = ['ptn {}'.format(lb + 1) for lb in df.columns]
    df.index.name = 'image'
    gc.collect(), time.sleep(1)
    return df


372
def process_experiment(path_expt, nb_jobs=NB_THREADS, path_img_rgb=PATH_IMAGES_RGB):
Jiri Borovec's avatar
Jiri Borovec committed
373 374 375 376
    """ process complete folder with experiment

    :param path_expt: str
    """
Jiri Borovec's avatar
Jiri Borovec committed
377
    logging.info('Experiment folder: \n "%s"', path_expt)
Jiri Borovec's avatar
Jiri Borovec committed
378
    dict_params = load_config_json(path_expt)
379 380 381
    dir_im = os.path.basename(dict_params['path_in'])
    dict_params['path_rgb'] = os.path.join(path_img_rgb,
                                           dir_im.replace('_segm_reg_binary', '_RGB_reg'))
Jiri Borovec's avatar
Jiri Borovec committed
382 383 384
    atlas_names = [os.path.basename(p) for p
                   in glob.glob(os.path.join(path_expt, PREFIX_ATLAS + '*.png'))]
    list_csv = [p for p in glob.glob(os.path.join(path_expt, PREFIX_ENCODE + '*.csv'))
Jiri Borovec's avatar
Jiri Borovec committed
385
                if not p.endswith(POSIX_CSV_NEW)]
Jiri Borovec's avatar
Jiri Borovec committed
386
    df_diffs_all = pd.DataFrame()
387
    for i, path_csv in enumerate(list_csv):
Jiri Borovec's avatar
Jiri Borovec committed
388 389
        name_csv = os.path.basename(path_csv)
        name_atlas = find_relevant_atlas(name_csv, atlas_names)
390 391
        logging.info('# %i / %i for Atlas: "%s" -> Encoding: "%s"',
                     i + 1, len(list_csv), name_atlas, name_csv)
392 393
        if name_atlas is None:
            continue
Jiri Borovec's avatar
Jiri Borovec committed
394 395 396
        # load the atlas
        path_atlas = os.path.join(path_expt, name_atlas)
        img_atlas = load_atlas_image(path_atlas)
Jiri Borovec's avatar
Jiri Borovec committed
397 398
        df_encode = pd.DataFrame.from_csv(path_csv)
        df_diff = perform_reconstruction_mproc(dict_params, name_csv.replace('.csv', ''),
399
                                               os.path.splitext(name_atlas)[0],
Jiri Borovec's avatar
Jiri Borovec committed
400
                                               df_encode, img_atlas, nb_jobs, VISUAL)
Jiri Borovec's avatar
Jiri Borovec committed
401
        df_diffs_all = pd.concat([df_diffs_all, df_diff], axis=1)
Jiri Borovec's avatar
Jiri Borovec committed
402
    df_diffs_all.to_csv(os.path.join(path_expt, CSV_RECONT_DIFF))
Jiri Borovec's avatar
Jiri Borovec committed
403
    if len(df_diffs_all) > 0:
Jiri Borovec's avatar
Jiri Borovec committed
404 405 406
        df_res = df_diffs_all.describe().transpose()[
            ['count', 'mean', 'std', 'min', 'max']]
        logging.info(repr(df_res))
Jiri Borovec's avatar
Jiri Borovec committed
407
        with open(os.path.join(path_expt, TXT_RECONT_DIFF), 'w') as fp:
Jiri Borovec's avatar
Jiri Borovec committed
408
            fp.write(repr(df_res))
Jiri Borovec's avatar
Jiri Borovec committed
409
    else:
Jiri Borovec's avatar
Jiri Borovec committed
410
        logging.error('no result parsed!')
Jiri Borovec's avatar
Jiri Borovec committed
411 412


Jiri Borovec's avatar
Jiri Borovec committed
413 414
def main():
    """ process complete list of experiments """
Jiri Borovec's avatar
Jiri Borovec committed
415
    logging.basicConfig(level=logging.INFO)
Jiri Borovec's avatar
Jiri Borovec committed
416
    logging.info('running...')
Jiri Borovec's avatar
Jiri Borovec committed
417

Jiri Borovec's avatar
Jiri Borovec committed
418 419 420 421 422 423
    arg_params = parse_arg_params(create_args_parser())
    logging.info('PARAMS: \n%s', '\n'.join(['"{}": \n\t {}'.format(k, v)
                                            for k, v in arg_params.iteritems()]))
    list_expt = [os.path.join(arg_params['path_in'], n) for n in arg_params['names_expt']]
    assert len(list_expt) > 0, 'No experiments found!'

424 425 426
    # tqdm_bar = tqdm.tqdm(total=len(list_expt))
    for i, path_expt in enumerate(list_expt):
        logging.info('experiment %i / %i -> %s', i + 1, len(list_expt), path_expt)
Jiri Borovec's avatar
Jiri Borovec committed
427 428
        process_experiment(path_expt, arg_params['nb_jobs'])
        gc.collect(), time.sleep(1)
429
        # tqdm_bar.update(1)
Jiri Borovec's avatar
Jiri Borovec committed
430 431

    logging.info('DONE')
Jiri Borovec's avatar
Jiri Borovec committed
432 433 434 435


if __name__ == '__main__':
    main()