Commit 9b20fce0 authored by Jiri Borovec's avatar Jiri Borovec

fix, not running

parent 15eae51e
......@@ -10,8 +10,11 @@ import os
import glob
import json
import logging
import time
import gc
import multiprocessing as mproc
import tqdm
import numpy as np
import pandas as pd
from PIL import Image
......@@ -22,7 +25,7 @@ import pattern_weights as ptn_weight
logger = logging.getLogger(__name__)
RUN_DEBUG = False
VISUAL = True
VISUAL = False
PATH_BASE = '/datagrid/Medical/microscopy/drosophila/'
if RUN_DEBUG:
......@@ -31,13 +34,17 @@ if RUN_DEBUG:
SAMPLE_PATH_EXPERIMENT = os.path.join(PATH_EXPERIMENTS,
'ExperimentALPE_mp_real_type_3_segm_reg_binary_gene_ssmall_20160507-000427')
else:
NB_THREADS = int(mproc.cpu_count() * .8)
PATH_EXPERIMENTS = os.path.join(PATH_BASE, 'TEMPORARY', 'experiments_APD_new')
NB_THREADS = int(mproc.cpu_count() * .9)
PATH_EXPERIMENTS = os.path.join(PATH_BASE, 'TEMPORARY', 'experiments_APDL_real')
SAMPLE_PATH_EXPERIMENT = ''
NAME_CONFIG = 'config.json'
PREFIX_ATLAS = 'atlas_'
PREFIX_ENCODE = 'encoding_'
PREFIX_RECONST = 'reconstruct_'
CSV_RECONT_DIFF = 'reconstruction_diff.csv'
TXT_RECONT_DIFF = 'reconstruction_diff.txt'
POSIX_CSV_SKIP = '_gene.csv'
def export_fig_reconstruction(path_out, name, segm_orig, segm_reconst, img_atlas):
......@@ -174,6 +181,7 @@ def load_segmentation(dict_params, img_name):
# # encoding from the input csv
# return compute_reconstruction(*mp_tuple)
def wrapper_reconstruction(mp_tuple):
# recompute encoding and then does the reconstruction
dict_params, path_out, img_atlas, weights, img_name = mp_tuple
......@@ -194,14 +202,15 @@ def export_fig_atlas(path_out, name, img_atlas):
plt.close(fig)
def perform_reconstruction_mproc(dict_params, name_csv, df_encode, img_atlas):
def perform_reconstruction_mproc(dict_params, name_csv, df_encode, img_atlas,
nb_jobs=NB_THREADS):
path_out = os.path.join(dict_params['path_exp'],
name_csv.replace(PREFIX_ENCODE, PREFIX_RECONST))
if not os.path.exists(path_out):
os.mkdir(path_out)
if VISUAL:
export_fig_atlas(path_out, 'atlas', img_atlas)
list_patterns = [col for col in df_encode.columns if col.startswith('ptn ')]
list_patterns = [col for col in df_encode.columns if col.startswith('ptn')]
logger.debug('list of pattern names: %s', repr(list_patterns))
list_idxs = [int(col[3:]) for col in list_patterns]
assert list_idxs == sorted(list_idxs)
......@@ -210,9 +219,9 @@ def perform_reconstruction_mproc(dict_params, name_csv, df_encode, img_atlas):
mp_tuples = ((dict_params, path_out, img_atlas, row[list_patterns].values, idx)
for idx, row in df_encode.iterrows())
if NB_THREADS > 1:
logger.debug('computing %i samples in %i threads', len(df_encode), NB_THREADS)
mproc_pool = mproc.Pool(NB_THREADS)
if nb_jobs > 1:
logger.debug('computing %i samples in %i threads', len(df_encode), nb_jobs)
mproc_pool = mproc.Pool(nb_jobs)
results = mproc_pool.map(wrapper_reconstruction, mp_tuples)
mproc_pool.close()
mproc_pool.join()
......@@ -225,18 +234,37 @@ def perform_reconstruction_mproc(dict_params, name_csv, df_encode, img_atlas):
return df_diff
def process_experiment(path_expt=SAMPLE_PATH_EXPERIMENT):
def load_config_json(path_expt):
with open(os.path.join(path_expt, NAME_CONFIG), 'r') as fp:
dict_params = json.load(fp)
# in case it was moved somewhere else
dict_params['path_exp'] = path_expt
for n in ['out_path', 'exp_path']:
if n in dict_params:
dict_params.pop(n)
dict_params['path_out'] = os.path.dirname(path_expt)
if 'in_path' in dict_params:
dict_params['path_in'] = dict_params.pop('in_path')
if 'sub_dataset' in dict_params:
dict_params['path_in'] = os.path.join(dict_params['path_in'],
dict_params.pop('dataset'))
dict_params['dataset'] = dict_params.pop('sub_dataset')
with open(os.path.join(path_expt, NAME_CONFIG), 'w') as fp:
json.dump(dict_params, fp)
return dict_params
def process_experiment(path_expt):
""" process complete folder with experiment
:param path_expt: str
"""
logger.info('Experiment folder: \n "%s"', path_expt)
with open(os.path.join(path_expt, NAME_CONFIG), 'r') as fp:
dict_params = json.load(fp)
dict_params = load_config_json(path_expt)
atlas_names = [os.path.basename(p) for p
in glob.glob(os.path.join(path_expt, PREFIX_ATLAS + '*.png'))]
list_csv = [p for p in glob.glob(os.path.join(path_expt, PREFIX_ENCODE + '*.csv'))
if not p.endswith('_gene.csv')]
if not p.endswith(POSIX_CSV_SKIP)]
df_diffs_all = pd.DataFrame()
for path_csv in list_csv:
name_csv = os.path.basename(path_csv)
......@@ -249,38 +277,38 @@ def process_experiment(path_expt=SAMPLE_PATH_EXPERIMENT):
df_diff = perform_reconstruction_mproc(dict_params, name_csv.replace('.csv', ''),
df_encode, img_atlas)
df_diffs_all = pd.concat([df_diffs_all, df_diff], axis=1)
df_diffs_all.to_csv(os.path.join(path_expt, 'reconstruction_diff.csv'))
df_diffs_all.to_csv(os.path.join(path_expt, CSV_RECONT_DIFF))
if len(df_diffs_all) > 0:
df_res = df_diffs_all.describe().transpose()[['count', 'mean', 'std', 'min', 'max']]
logger.info(repr(df_res))
with open(os.path.join(path_expt, 'reconstruction_diff.txt'), 'w') as fp:
with open(os.path.join(path_expt, TXT_RECONT_DIFF), 'w') as fp:
fp.write(repr(df_res))
else:
logger.error('no result parsed!')
def main(path_base=PATH_EXPERIMENTS):
""" process complete liat of experiments
def main(path_data=PATH_EXPERIMENTS):
""" process complete list of experiments
:param path_base: str
:param path_data: str
"""
list_expt = [p for p in glob.glob(os.path.join(path_base, '*'))
if os.path.isdir(p)]
for i, path_expt in enumerate(list_expt):
logger.info('processing experiment %i / %i', i + 1, len(list_expt))
process_experiment(path_expt)
def main():
logging.basicConfig(level=logging.INFO)
if RUN_DEBUG:
logging.basicConfig(level=logging.DEBUG)
logger.info('running...')
if RUN_DEBUG:
process_experiment()
logging.basicConfig(level=logging.DEBUG)
process_experiment(SAMPLE_PATH_EXPERIMENT)
else:
main()
assert os.path.exists(path_data)
list_expt = sorted([p for p in glob.glob(os.path.join(path_data, '*'))
if os.path.isdir(p)])
logging.info('found %i in folder "%s"', len(list_expt), path_data)
tqdm_bar = tqdm.tqdm(total=len(list_expt))
for path_expt in list_expt:
process_experiment(path_expt)
gc.collect(), time.sleep(1)
tqdm_bar.update(1)
logger.info('DONE')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment