https://gitlab.limos.fr/iia_lulibral/experiment-results
Raw File
Tip revision: 00ab06ddf7225f8036ee6261bb5d474f9e40befb authored by Default on 15 July 2021, 15:21:10 UTC
add warp experiment results
Tip revision: 00ab06d
gen_recap.py
#!/usr/bin/python3
from helper import read_json, extract_labels, extract_v

        

def gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes):
    model_variants = []
    model_score = {}
    for x in range(0,5+1):
        for a,b in [(0,0),(1,0), (1,1)]:
            model_variants.append((x,a,b))
            model_score[(x,a,b)] = {
                'nb_solved_opt': 0,
                'nb_solved_enum': 0,
                'nb_deviations_vopt': 0,
                'total_deviations_vopt': 0,
                'nb_sols': 0
            }
    # result extraction
    insts = {}
    cryptosystems = {}
    for name,rmin,rmax in inst_classes:
        for r in range(rmin,rmax+1):
            crypto_name = name+"_{}".format(r)
            cryptosystems[crypto_name] = {"vbest":0}
            for x,a,b in model_variants:
                p = read_json(res_picat+  "{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
                g = read_json(res_gurobi+ "{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
                c = read_json(res_chuffed+"{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
                ge = read_json(res_gecode+"{}_{}_{}_{}_{}.json.mzn".format(name, r,x,a,b))
                for st in ["v", "nb_sols"]:
                    st_set = set(map(lambda e: e[st], filter(lambda e: st in e, [p,g,c,ge])))
                    if len(st_set) > 1:
                        print("ERROR: {} {} {} {} {} model has different {}:\t{}\t| picat:{}\tgurobi:{},\tchuffed:{},\tgecode:{}".format(name, r, x, a, b, st, st_set, p.get(st,-1),g.get(st,-1),c.get(st,-1),ge.get(st,-1)))
                    insts["{}_{}_{}_{}_{}".format(name, r,x,a,b)] = {
                        'picat': p, 'gurobi': g, 'chuffed': c, 'gecode': ge,
                    }
                    v_set = list(map(lambda e: e["v"], filter(lambda e: "v" in e, [p,g,c,ge])))
                    if len(v_set) > 0:
                        v_model = list(map(lambda e: e["v"], filter(lambda e: "v" in e, [p,g,c,ge])))[0]
                        cryptosystems[crypto_name]["vbest"] = max(
                            cryptosystems[crypto_name]["vbest"],
                            v_model
                        )
    # print(insts)
    # print(cryptosystems)
    # extract model variant info
    for name,rmin,rmax in inst_classes:
        for r in range(rmin,rmax+1):
            crypto_name = name+"_{}".format(r)
            vbest = cryptosystems[crypto_name]["vbest"]
            # print("{}\tv*: {}".format(crypto_name, vbest))
            for x,a,b in model_variants:
                iname = "{}_{}_{}_{}_{}".format(name, r,x,a,b)
                p,g,c,ge = insts[iname]["picat"],insts[iname]["gurobi"],insts[iname]["chuffed"],insts[iname]["gecode"]
                v_set = list(set(map(lambda e: e["v"], filter(lambda e: "v" in e, [p,g,c,ge]))))
                nb_set = v_model = list(set(map(lambda e: e["nb_sols"], filter(lambda e: "nb_sols" in e, [p,g,c,ge]))))
                if len(v_set) == 1:
                    model_score[(x,a,b)]["nb_solved_opt"] += 1
                    if v_set[0] < cryptosystems[crypto_name]["vbest"]:
                        model_score[(x,a,b)]["nb_deviations_vopt"] += 1
                        model_score[(x,a,b)]["total_deviations_vopt"] += cryptosystems[crypto_name]["vbest"]-v_set[0]
                elif len(v_set) > 1:
                    print("WARNING: error (v) in models for {}".format(crypto_name))
                if len(nb_set) == 1:
                    model_score[(x,a,b)]["nb_solved_enum"] += 1
                    model_score[(x,a,b)]["nb_sols"] += nb_set[0]
                elif len(nb_set) > 1:
                    print("WARNING: error (nb_sol) in models for {}".format(crypto_name))
    # print results
    res = {}
    # print("\\begin{tabular}{l|lll}")
    # print("model & \\#dev& \\#opt \\\\")
    # print("\\hline")
    for a,b in [(0,0),(1,0),(1,1)]:
        for x in range(0,5+1):
            if model_score[(x,a,b)]["nb_solved_opt"] > 0:
                res["x{} {} {}".format(x,"diff" if a == 1 else "","tr" if b == 1 else "")] = [
                    model_score[(x,a,b)]["nb_deviations_vopt"],
                    model_score[(x,a,b)]["nb_solved_opt"],
                    model_score[(x,a,b)]["nb_solved_enum"]
                ]
    return res
    # print("\\begin{tabular}{l|lll}")
    # print("model & \\#dev& \\#opt \\\\")
    # print("\\hline")
    # for a,b in [(0,0),(1,0),(1,1)]:
    #     for x in range(0,5+1):
    #         if model_score[(x,a,b)]["nb_solved_opt"] > 0:
    #             print("x{} {} {} & {} & {} & {} \\\\".format(
    #                 x,"diff" if a == 1 else "","tr" if b == 1 else "",
    #                 model_score[(x,a,b)]["nb_deviations_vopt"],
    #                 model_score[(x,a,b)]["nb_solved_opt"],
    #                 model_score[(x,a,b)]["nb_solved_enum"]
    #             ))
    #     print("\\hline")
    # print("\\end{tabular}")



if __name__ == "__main__":
    res_picat   = "res_picat/"
    res_gurobi  = "res_gurobi/"
    res_chuffed = "res_chuffed/"
    res_gecode  = "res_gecode/"

    ############ MIDORI
    inst_classes = [
        ("midori_sk", 3, 5),
        ("midori_64", 3, 16),
        ("midori_128", 3, 20),
    ]
    nb_midori = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
    table_midori = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
    ############## AES
    inst_classes = [
        ("aes_sk", 3, 4),
        ("rijndael_4_4", 3, 5),
        ("rijndael_4_6", 3, 10),
        ("rijndael_4_8", 3, 14),
    ]
    nb_aes = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
    table_aes = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
    ############## SKINNY
    inst_classes = [
        ("skinny_64_0", 3, 30),
        ("skinny_64_1", 3, 30),
    ]
    nb_skinny = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
    table_skinny = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
    ############## CRAFT
    inst_classes = [
        ("craft_sk", 2,20),
        ("craft_rk", 2,20),
    ]
    nb_craft = sum(map(lambda e: e[2]-e[1]+1, inst_classes))
    table_craft = gen_recap(res_picat, res_gurobi, res_chuffed, res_gecode, inst_classes)
    print("\\begin{tabular}{l|lll|lll|lll|lll}")
    print(" & \\multicolumn{3}{c}{Midori ("+str(nb_midori)+")} & \\multicolumn{3}{c}{AES ("+str(nb_aes)+")} & \\multicolumn{3}{c}{SKINNY ("+str(nb_skinny)+")} & \\multicolumn{3}{c}{CRAFT ("+str(nb_craft)+")} \\\\")
    print("model \t&\t \\#d \t&\t \\#o \t&\t \\#e \t&\t \\#d \t&\t \\#o \t&\t \\#e \t&\t \\#d \t&\t \\#o \t&\t \\#e \t&\t \\#d \t&\t \\#o \t&\t \\#e \\\\")
    for a,b in [(0,0),(1,0)]:
        print("\\hline")
        for x in range(0,5+1):
            modelname = "x{} {} {}".format(x,"diff" if a == 1 else "","tr" if b == 1 else "")
            line = ""
            table_lists = [table_midori, table_aes, table_skinny, table_craft]
            for table in table_lists:
                if modelname in table:
                    line += "\t&\t" + "\t&\t".join([str(e) for e in table[modelname]])
                else:
                    na_symbol = "--"
                    line += " \t&\t {} \t&\t {} \t&\t {}".format(na_symbol, na_symbol, na_symbol)
            print(modelname.replace("x","$x_{max}$=") + line + " \\\\")
    print("\\end{tabular}")
back to top