https://gitlab.limos.fr/iia_lulibral/experiment-results
Tip revision: 00ab06ddf7225f8036ee6261bb5d474f9e40befb authored by Default on 15 July 2021, 15:21:10 UTC
add warp experiment results
add warp experiment results
Tip revision: 00ab06d
gen_recap.py
#!/usr/bin/python3
from helper import read_json, extract_labels, extract_v
def gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes):
model_variants = []
model_score = {}
for x in range(0,5+1):
for a,b in [(0,0),(1,0), (1,1)]:
model_variants.append((x,a,b))
model_score[(x,a,b)] = {
'nb_solved_opt': 0,
'nb_solved_enum': 0,
'nb_deviations_vopt': 0,
'total_deviations_vopt': 0,
'nb_sols': 0
}
# result extraction
insts = {}
cryptosystems = {}
for name,rmin,rmax in inst_classes:
for r in range(rmin,rmax+1):
crypto_name = name+"_{}".format(r)
cryptosystems[crypto_name] = {"vbest":0}
for x,a,b in model_variants:
p = read_json(res_picat+ "{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
g = read_json(res_gurobi+ "{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
c = read_json(res_chuffed+"{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
ge = read_json(res_gecode+"{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
for st in ["v", "nb_sols"]:
st_set = set(map(lambda e: e[st], filter(lambda e: st in e, [p,g,c,ge])))
if len(st_set) > 1:
print("ERROR: {} {} {} {} {} model has different {}:\t{}\t| picat:{}\tgurobi:{},\tchuffed:{},\tgecode:{}".format(name, r, x, a, b, st, st_set, p.get(st,-1),g.get(st,-1),c.get(st,-1),ge.get(st,-1)))
insts["{}_{}_{}_{}_{}".format(name, r,x,a,b)] = {
'picat': p, 'gurobi': g, 'chuffed': c, 'gecode': ge,
}
v_set = list(map(lambda e: e["v"], filter(lambda e: "v" in e, [p,g,c,ge])))
if len(v_set) > 0:
v_model = list(map(lambda e: e["v"], filter(lambda e: "v" in e, [p,g,c,ge])))[0]
cryptosystems[crypto_name]["vbest"] = max(
cryptosystems[crypto_name]["vbest"],
v_model
)
# print(insts)
# print(cryptosystems)
# extract model variant info
for name,rmin,rmax in inst_classes:
for r in range(rmin,rmax+1):
crypto_name = name+"_{}".format(r)
vbest = cryptosystems[crypto_name]["vbest"]
# print("{}\tv*: {}".format(crypto_name, vbest))
for x,a,b in model_variants:
iname = "{}_{}_{}_{}_{}".format(name, r,x,a,b)
p,g,c,ge = insts[iname]["picat"],insts[iname]["gurobi"],insts[iname]["chuffed"],insts[iname]["gecode"]
v_set = list(set(map(lambda e: e["v"], filter(lambda e: "v" in e, [p,g,c,ge]))))
nb_set = v_model = list(set(map(lambda e: e["nb_sols"], filter(lambda e: "nb_sols" in e, [p,g,c,ge]))))
if len(v_set) == 1:
model_score[(x,a,b)]["nb_solved_opt"] += 1
if v_set[0] < cryptosystems[crypto_name]["vbest"]:
model_score[(x,a,b)]["nb_deviations_vopt"] += 1
model_score[(x,a,b)]["total_deviations_vopt"] += cryptosystems[crypto_name]["vbest"]-v_set[0]
elif len(v_set) > 1:
print("WARNING: error (v) in models for {}".format(crypto_name))
if len(nb_set) == 1:
model_score[(x,a,b)]["nb_solved_enum"] += 1
model_score[(x,a,b)]["nb_sols"] += nb_set[0]
elif len(nb_set) > 1:
print("WARNING: error (nb_sol) in models for {}".format(crypto_name))
# print results
res = {}
# print("\\begin{tabular}{l|lll}")
# print("model & \\#dev& \\#opt \\\\")
# print("\\hline")
for a,b in [(0,0),(1,0),(1,1)]:
for x in range(0,5+1):
if model_score[(x,a,b)]["nb_solved_opt"] > 0:
res["x{} {} {}".format(x,"diff" if a == 1 else "","tr" if b == 1 else "")] = [
model_score[(x,a,b)]["nb_deviations_vopt"],
model_score[(x,a,b)]["nb_solved_opt"],
model_score[(x,a,b)]["nb_solved_enum"]
]
return res
# print("\\begin{tabular}{l|lll}")
# print("model & \\#dev& \\#opt \\\\")
# print("\\hline")
# for a,b in [(0,0),(1,0),(1,1)]:
# for x in range(0,5+1):
# if model_score[(x,a,b)]["nb_solved_opt"] > 0:
# print("x{} {} {} & {} & {} & {} \\\\".format(
# x,"diff" if a == 1 else "","tr" if b == 1 else "",
# model_score[(x,a,b)]["nb_deviations_vopt"],
# model_score[(x,a,b)]["nb_solved_opt"],
# model_score[(x,a,b)]["nb_solved_enum"]
# ))
# print("\\hline")
# print("\\end{tabular}")
if __name__ == "__main__":
res_picat = "res_picat/"
res_gurobi = "res_gurobi/"
res_chuffed = "res_chuffed/"
res_gecode = "res_gecode/"
############ MIDORI
inst_classes = [
("midori_sk", 3, 5),
("midori_64", 3, 16),
("midori_128", 3, 20),
]
nb_midori = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
table_midori = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
############## AES
inst_classes = [
("aes_sk", 3, 4),
("rijndael_4_4", 3, 5),
("rijndael_4_6", 3, 10),
("rijndael_4_8", 3, 14),
]
nb_aes = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
table_aes = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
############## SKINNY
inst_classes = [
("skinny_64_0", 3, 30),
("skinny_64_1", 3, 30),
]
nb_skinny = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
table_skinny = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
############## CRAFT
inst_classes = [
("craft_sk", 2,20),
("craft_rk", 2,20),
]
nb_craft = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
table_craft = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
print("\\begin{tabular}{l|lll|lll|lll|lll}")
print(" & \\multicolumn{3}{c}{Midori ("+str(nb_midori)+")} & \\multicolumn{3}{c}{AES ("+str(nb_aes)+")} & \\multicolumn{3}{c}{SKINNY ("+str(nb_skinny)+")} & \\multicolumn{3}{c}{CRAFT ("+str(nb_craft)+")} \\\\")
print("model \t&\t \\#d \t&\t \\#o \t&\t \\#e \t&\t \\#d \t&\t \\#o \t&\t \\#e \t&\t \\#d \t&\t \\#o \t&\t \\#e \t&\t \\#d \t&\t \\#o \t&\t \\#e \\\\")
for a,b in [(0,0),(1,0)]:
print("\\hline")
for x in range(0,5+1):
modelname = "x{} {} {}".format(x,"diff" if a == 1 else "","tr" if b == 1 else "")
line = ""
table_lists = [table_midori, table_aes, table_skinny, table_craft]
for table in table_lists:
if modelname in table:
line += "\t&\t" + "\t&\t".join([str(e) for e in table[modelname]])
else:
na_symbol = "--"
line += " \t&\t {} \t&\t {} \t&\t {}".format(na_symbol, na_symbol, na_symbol)
print(modelname.replace("x","$x_{max}$=") + line + " \\\\")
print("\\end{tabular}")