Commit fe862c7e authored by Martin Řepa's avatar Martin Řepa

Prepare first experiment

parent aed95e2c
use_blocking_model: True
legacy_folder: /home/ignac/experiments/game_theory_model_blocking/legacy1
legacy_folder: /home/ignac/experiments/game_theory_model_blocking/legacy1 # Change in regards with configuration
num_of_experiments: 2
nn_epochs: 2000
......
import os
import pickle
import sys
import time
from dataclasses import dataclass
from os.path import dirname
......@@ -65,15 +66,7 @@ def get_root_conf(conf_of_conf: dict) -> RootConfig:
return conf
if __name__ == "__main__":
print('Starting game theory blocking model experiment')
experiment_conf = get_configuration()
base_dir = experiment_conf["legacy_folder"]
if not os.path.exists(base_dir):
print(f'Creating base dir {base_dir}')
os.mkdir(base_dir)
def main(experiment_conf, base_dir):
conf = get_root_conf(experiment_conf)
repetitions = experiment_conf['num_of_experiments']
......@@ -133,3 +126,19 @@ if __name__ == "__main__":
conf.model_conf.defender_conf.nn_conf.loss_function = None
pickle.dump(conf, file)
print('File saved.')
if __name__ == "__main__":
print('Starting game theory blocking model experiment')
experiment_conf = get_configuration()
base_dir = experiment_conf["legacy_folder"]
if not os.path.exists(base_dir):
print(f'Creating base dir {base_dir}')
os.mkdir(base_dir)
log_file = f'{base_dir}/log'
with open(log_file, 'w') as log:
sys.stderr = log
sys.stdout = log
main(experiment_conf, base_dir)
use_blocking_model: False
legacy_folder: /home/ignac/experiments/game_theory_model_latency/legacy1
num_of_experiments: 2
legacy_folder: /home/ignac/experiments/game_theory_model_latency/legacy1 # Change in regards with configuration
num_of_experiments: 5
nn_epochs: 2000
nn_epochs: 10
i_a: 1
i_d: 2
benign_ratio: 10000
......
import os
import pickle
import sys
import time
from dataclasses import dataclass
from os.path import dirname
......@@ -65,16 +66,7 @@ def get_root_conf(conf_of_conf: dict) -> RootConfig:
conf.model_conf.defender_conf.nn_conf.epochs = conf_of_conf['nn_epochs']
return conf
if __name__ == "__main__":
print('Starting game theory latency model experiment')
experiment_conf = get_configuration()
base_dir = experiment_conf["legacy_folder"]
if not os.path.exists(base_dir):
print(f'Creating base dir {base_dir}')
os.mkdir(base_dir)
def main(experiment_conf, base_dir):
conf = get_root_conf(experiment_conf)
repetitions = experiment_conf['num_of_experiments']
......@@ -134,3 +126,21 @@ if __name__ == "__main__":
conf.model_conf.defender_conf.nn_conf.loss_function = None
pickle.dump(conf, file)
print('File saved.')
if __name__ == "__main__":
print('Starting game theory latency model experiment')
experiment_conf = get_configuration()
base_dir = experiment_conf["legacy_folder"]
if not os.path.exists(base_dir):
print(f'Creating base dir {base_dir}')
os.mkdir(base_dir)
log_file = f'{base_dir}/log'
with open(log_file, 'w') as log:
sys.stderr = log
sys.stdout = log
main(experiment_conf, base_dir)
experiments_per_setup: 1 # 6
legacy_folder: /home/ignac/experiments/learning_epochs/legacy1
experiments_per_setup: 3 # 6
legacy_folder: /home/ignac/experiments/learning_epochs/2000to14000by4000_3times # Change in regards with configuration
epochs:
lower_bound: 2000
number_of_steps: 2 # 6-8 should be enough
upper_bound: 5000 # 30000
number_of_steps: 4 # 8 (6-8 should be enough)
upper_bound: 14000 # 30000
conf:
i_a: 1
i_d: 2
benign_ratio: 10000
benign_ratio: 5
data_file: normal_distribution_experiments.csv
import os
import pickle
import sys
import time
from os.path import dirname
from pathlib import Path
......@@ -110,15 +111,7 @@ def get_root_conf(conf_of_conf: dict) -> RootConfig:
return conf
if __name__ == "__main__":
print('Starting learning epochs experiment')
experiment_conf = get_configuration()
base_dir = experiment_conf["legacy_folder"]
if not os.path.exists(base_dir):
print(f'Creating base dir {base_dir}')
os.mkdir(base_dir)
def main(experiment_conf, base_dir):
conf = get_root_conf(experiment_conf['conf'])
cur_epochs = experiment_conf['epochs']['lower_bound']
......@@ -161,3 +154,20 @@ if __name__ == "__main__":
conf.model_conf.defender_conf.nn_conf.loss_function = None
pickle.dump(conf, file)
print('File saved.')
if __name__ == "__main__":
print('Starting learning epochs experiment')
experiment_conf = get_configuration()
base_dir = experiment_conf["legacy_folder"]
if not os.path.exists(base_dir):
print(f'Creating base dir {base_dir}')
os.mkdir(base_dir)
log_file = f'{base_dir}/log'
with open(log_file, 'w') as log:
sys.stderr = log
sys.stdout = log
main(experiment_conf, base_dir)
......@@ -21,8 +21,9 @@ density = 2 # 1 is low | 2 is high
benign_ratio = 5
data_file = 'normal_distribution_experiments.csv'
run_times = 1
legacy_folder = '/home/ignac/experiments/lp_blocking/legacy1'
run_times = 3
legacy_folder = f'/home/ignac/experiments/lp_blocking/' \
f'feat{features_num}_br{benign_ratio}_dens{density}_{run_times}times'
@dataclass
......@@ -76,9 +77,6 @@ def solve_with_lp(actions_attacker: List,
actions_defender: List,
benign_data_prob: dict):
print('Going to solve with LP')
print(f'Attacker\'s actions: {actions_attacker}')
print(f'Defender\'s actions: {actions_defender}')
print(f'Benign data probabilities are: {benign_data_prob}')
# Create LP problem
m = pulp.LpProblem("Zero sum game", pulp.LpMinimize)
......@@ -90,21 +88,21 @@ def solve_with_lp(actions_attacker: List,
# Player two probabilities vector
print("Defining defenders probabilities...")
probs_defender = []
for action in defender_actions:
for action in actions_defender:
probs_point = pulp.LpVariable(f'p({action[0]},{action[1]})', 0, 1)
probs_defender.append(probs_point)
fp_cost = 0
num_of_fp = 0
for action, prob in zip(defender_actions, probs_defender):
for action, prob in zip(actions_defender, probs_defender):
cur_fp = prob * benign_data_prob[action]
num_of_fp += cur_fp
fp_cost += cur_fp * benign_ratio
print("Defining main constraint...")
constraints = []
for i in range(len(attacker_actions)):
sum = [fp_cost, (1 - probs_defender[i]) * utility(attacker_actions[i])]
for i in range(len(actions_attacker)):
sum = [fp_cost, (1 - probs_defender[i]) * utility(actions_attacker[i])]
constraints.append(pulp.lpSum(sum) <= v)
for c in constraints:
m += c
......@@ -119,18 +117,18 @@ def solve_with_lp(actions_attacker: List,
for i in range(len(constraints)):
prob = abs(constraints[i].pi)
if prob == 0: continue
print(f'Action {attacker_actions[i]} -> {prob}')
print(f'Action {actions_attacker[i]} -> {prob}')
print(f'Deffender\'s probabilities to block:')
for i in range(len(probs_defender)):
print(f'Action {attacker_actions[i]} -> {probs_defender[i].varValue}')
print(f'Action {actions_attacker[i]} -> {probs_defender[i].varValue}')
# Return value and probabilities
return v.varValue, [abs(c.pi) for c in constraints], \
[prob.varValue for prob in probs_defender]
def calc_fals_positives(p2_actions, p2_probs):
def calc_fals_positives(p2_actions, p2_probs, benign_probs):
fp_cost = 0
for action, prob in zip(p2_actions, p2_probs):
if prob is None: prob = 0
......@@ -138,7 +136,7 @@ def calc_fals_positives(p2_actions, p2_probs):
return fp_cost * benign_ratio
if __name__ == "__main__":
def main():
# Get actions
attacker_actions = create_actions()
defender_actions = create_actions()
......@@ -156,7 +154,7 @@ if __name__ == "__main__":
game_val, attacker_probs, probs_defender = res
time_takens.append(time.time() - start)
fp_part = calc_fals_positives(defender_actions, probs_defender)
fp_part = calc_fals_positives(defender_actions, probs_defender, benign_probs)
final_experiment = Result(game_val,
fp_part,
......@@ -167,11 +165,6 @@ if __name__ == "__main__":
time_takens)
print("Experiment done.")
# Create base folder if it does not exist
if not os.path.exists(legacy_folder):
print(f'Creating base dir {legacy_folder}')
os.mkdir(legacy_folder)
# ----------------- Store final data -----------------
experiment_data_file = f'{legacy_folder}/data'
print(f'Storing final data to {experiment_data_file} file.')
......@@ -194,3 +187,20 @@ if __name__ == "__main__":
pickle.dump(final_experiment, file)
print('File saved')
print('Recapitulation:')
print(f'Final value {game_val}')
print(backup_conf_file)
if __name__ == "__main__":
# Create base folder if it does not exist
if not os.path.exists(legacy_folder):
print(f'Creating base dir {legacy_folder}')
os.mkdir(legacy_folder)
log_file = f'{legacy_folder}/log'
with open(log_file, 'w') as log:
sys.stderr = log
sys.stdout = log
main()
......@@ -12,18 +12,35 @@ import pulp
from src.data.loader import np_matrix_from_scored_csv
latencies, l = None, None
try:
latencies = sys.argv[1]
except IndexError:
latencies = 'big'
if latencies == 'big':
l = list(np.round(np.linspace(0, 1, 11), 2))
elif latencies == 'small':
l = list(np.linspace(0, 1, 5))
else:
print('Latencies is not small or big!')
sys.exit(6)
# Global "config" variables
features_num = 2
density = 2 # 1 is low | 2 is high
l = [.0, .25, .50, .75, 1.0] # [.0, .1, .2, .3, .4, .5, .6, .7, .8, .9, 1.]
i_a = 1
i_d = 2
benign_ratio = 5
data_file = 'normal_distribution_experiments.csv'
run_times = 1
legacy_folder = '/home/ignac/experiments/lp_latency/legacy1'
run_times = 3
legacy_folder = '/home/ignac/experiments/lp_latency' \
f'feat{features_num}_br{benign_ratio}_dens{density}_ia{i_a}' \
f'_id{i_d}_{run_times}times_latencies{latencies}'
@dataclass
......@@ -117,15 +134,15 @@ def solve_with_lp(actions_attacker: List,
return v.varValue, [abs(c.pi) for c in constraints], def_probs_final
def calc_fals_positives(p2_actions, p2_probs, loss):
def calc_fals_positives(p2_actions, p2_probs, loss, benign_probs):
fp_cost = 0
for action, prob in zip(p2_actions, p2_probs):
for j in range(len(l)):
fp_cost += loss(l[j])*benign_data_prob[tuple(action)]*prob[j]
fp_cost += loss(l[j])*benign_probs[tuple(action)]*prob[j]
return fp_cost * benign_ratio
if __name__ == "__main__":
def main():
attacker_actions = create_actions()
defender_actions = create_actions()
......@@ -149,7 +166,8 @@ if __name__ == "__main__":
game_val, attacker_probs, probs_defender = res
time_takens.append(time.time() - start)
fp_part = calc_fals_positives(defender_actions, probs_defender, fp_cost_loss)
fp_part = calc_fals_positives(defender_actions, probs_defender,
fp_cost_loss, benign_data_prob)
final_experiment = Result(game_val,
fp_part,
......@@ -160,11 +178,6 @@ if __name__ == "__main__":
l, time_takens)
print("Experiment done.")
# Create base folder if it does not exist
if not os.path.exists(legacy_folder):
print(f'Creating base dir {legacy_folder}')
os.mkdir(legacy_folder)
# ----------------- Store final data -----------------
experiment_data_file = f'{legacy_folder}/data'
print(f'Storing final data to {experiment_data_file} file.')
......@@ -190,3 +203,19 @@ if __name__ == "__main__":
pickle.dump(final_experiment, file)
print('File saved')
print('Recapitulation:')
print(f'Final value {game_val}')
print(backup_conf_file)
if __name__ == "__main__":
# Create base folder if it does not exist
if not os.path.exists(legacy_folder):
print(f'Creating base dir {legacy_folder}')
os.mkdir(legacy_folder)
log_file = f'{legacy_folder}/log'
with open(log_file, 'w') as log:
sys.stderr = log
sys.stdout = log
main()
......@@ -4,19 +4,22 @@ set -e
export PYTHONPATH=/app
# echo "Going to run learning epochs"
# python3.7 src/experiments/learning_epochs/learning_epochs.py &> /home/ignac/experiments/learning_epochs/log
echo "Going to run learning epochs"
python3.7 src/experiments/learning_epochs/learning_epochs.py
# echo "Going to run blocking GT model"
# python3.7 src/experiments/game_theory_blocking/run_blocking_model.py &> /home/ignac/experiments/game_theory_model_blocking/log
# python3.7 src/experiments/game_theory_blocking/run_blocking_model.py
# echo "Going to run latency GT model"
# python3.7 src/experiments/game_theory_latency/run_latency_model.py &> /home/ignac/experiments/game_theory_model_latency/log
# python3.7 src/experiments/game_theory_latency/run_latency_model.py
echo "Going to run lp blocking"
python3.7 src/experiments/lp_blocking/lp_blocking.py &> /home/ignac/experiments/lp_blocking/log
python3.7 src/experiments/lp_blocking/lp_blocking.py
echo "Going to run lp latency"
python3.7 src/experiments/lp_latency/lp_latency.py &> /home/ignac/experiments/lp_latency/log
echo "Going to run lp latency with big latencies"
python3.7 src/experiments/lp_latency/lp_latency.py
echo "Going to run lp latency with small latencies"
python3.7 src/experiments/lp_latency/lp_latency.py small
echo "All experiments are done..."
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment