Commit 86ab5ac5 authored by Martin Řepa's avatar Martin Řepa

Prepare game theory blocking model experiment

parent 98dbc187
use_blocking_model: True
legacy_folder: /home/ignac/experiments/game_theory_model_blocking/legacy1 # Change in regards with configuration
num_of_experiments: 2
nn_epochs: 2000
benign_ratio: 10000
features_num: 2
data_file: normal_distribution_experiments.csv
experiments_per_setup: 5 # 6
legacy_folder: /home/ignac/experiments/game_theory_model_blocking/2dimension_discrete_gradient_5times # Change in regards with configuration
attacker_mode: both # options are 'both', 'gradient', 'discrete'
conf:
attacker_epochs: 3000
defender_epochs: 4000
benign_ratio: 1
data_file: normal_distribution_experiments.csv
features: 2
......@@ -2,13 +2,13 @@ import os
import pickle
import sys
import time
from dataclasses import dataclass
from os.path import dirname
from pathlib import Path
from typing import List
import torch
import yaml
from dataclasses import dataclass
from src.config import RootConfig
from src.game import Game
......@@ -32,6 +32,7 @@ class DefenderAction:
@dataclass
class SubResult:
iterations: int
legacy_folder: str
time: float
zero_sum_game_value: float
......@@ -41,9 +42,15 @@ class SubResult:
defender_actions: List[DefenderAction]
@dataclass
class Setup:
name: str
results: List[SubResult]
@dataclass
class Experiment:
sub_results: List[SubResult]
setups: List[Setup]
def get_configuration() -> dict:
......@@ -53,33 +60,23 @@ def get_configuration() -> dict:
return yaml.load(content, Loader=yaml.FullLoader)
def get_root_conf(conf_of_conf: dict) -> RootConfig:
conf = RootConfig()
conf.debug = False
conf.plot_conf.plot_enabled = False
conf.model_conf.use_blocking_model = bool(conf_of_conf['use_blocking_model'])
conf.model_conf.set_ia_id_benign_ration(.5, .5, # W/e these values
conf_of_conf['benign_ratio'])
conf.model_conf.set_data_file(conf_of_conf['data_file'])
conf.model_conf.defender_conf.nn_conf.epochs = conf_of_conf['nn_epochs']
return conf
def main(experiment_conf, base_dir):
conf = get_root_conf(experiment_conf)
repetitions = experiment_conf['num_of_experiments']
def exec_new_setup(conf: RootConfig, folder: str, iterations: int, name: str) -> Setup:
sub_results = []
for i in range(repetitions):
print(f'Starting {i+1}. iteration.')
os.mkdir(f'{base_dir}/{i}')
i = 0
while i < iterations:
if not os.path.exists(f'{folder}/{i}'):
os.mkdir(f'{folder}/{i}')
print(f'Starting {i+1}. iteration of setup with {name} attacker')
start = time.time()
result = Game(conf).solve_game()
time_taken = time.time() - start
if result.iterations == 1:
continue
attacker_actions = []
for p1_action, p1_prob in zip(result.ordered_actions_p1,
result.probs_p1):
......@@ -95,9 +92,10 @@ def main(experiment_conf, base_dir):
p2_action.final_loss,
p2_action.final_fp_cost))
torch.save(p2_action.model.state_dict(),
f'{base_dir}/{i}/{p2_action.id}.pt')
f'{folder}/{i}/{p2_action.id}.pt')
sub_result = SubResult(str(i),
sub_result = SubResult(result.iterations,
str(i),
time_taken,
result.zero_sum_nash_val,
result.attacker_value,
......@@ -105,8 +103,61 @@ def main(experiment_conf, base_dir):
attacker_actions,
defender_actions)
sub_results.append(sub_result)
i += 1
final_setup = Setup(name, sub_results)
# ------------ Save backup data from this setup ---------
data_file = f'{folder}/backup_setup_data'
with open(data_file, 'wb') as file:
pickle.dump(final_setup, file)
# -------------------------------------------------------
return final_setup
def get_root_conf(conf_of_conf: dict) -> RootConfig:
conf = RootConfig()
conf.debug = False
conf.plot_conf.plot_enabled = False
conf.model_conf.attacker_conf.epochs = conf_of_conf['attacker_epochs']
conf.model_conf.defender_conf.nn_conf.epochs = conf_of_conf['defender_epochs']
conf.model_conf.features_count = conf_of_conf['features']
conf.model_conf.use_blocking_model = True
conf.model_conf.set_ia_id_benign_ration(1, 1, conf_of_conf['benign_ratio'])
conf.model_conf.set_data_file(conf_of_conf['data_file'])
return conf
done_experiment = Experiment(sub_results)
def main(experiment_conf, base_dir):
conf = get_root_conf(experiment_conf['conf'])
attacker_mode = experiment_conf['attacker_mode']
modes = None
if attacker_mode == 'both':
modes = [(True, 'gradient'), (False, 'discrete')]
elif attacker_mode == 'gradient':
modes = [(True, 'gradient')]
elif attacker_mode == 'discrete':
modes = [(False, 'discrete')]
else:
print('Invalid attacker mode: ' + str(attacker_mode))
exit(10)
setups = []
for use_gradient_descent, name in modes:
print(f'Let\'s do subexperiments for {name} attacker setup.')
folder = f'{base_dir}/{name}'
os.mkdir(folder)
print(f'Result is gonna be stored in {folder}.')
conf.model_conf.attacker_conf.use_gradient_descent = use_gradient_descent
setup = exec_new_setup(conf, folder,
experiment_conf['experiments_per_setup'], name)
setups.append(setup)
done_experiment = Experiment(setups)
print('Experiment done.')
# Save the result data
......@@ -142,3 +193,4 @@ if __name__ == "__main__":
sys.stderr = log
sys.stdout = log
main(experiment_conf, base_dir)
# main(experiment_conf, base_dir)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment