import json import os import os.path import datetime import xlwt import pprint import copy import numpy as np import matplotlib import matplotlib.pyplot as plt import matplotlib.colors as colors from PIL import Image # PREAMBLE, SCROLL DOWN FOR SETTINGS # Some of the experimental runs contained multiple descriptor types. # However, I found out later that some of these descriptor generation/testing implementations contained some bugs # Which rendered the results of specific descriptors, but not the others, of specific result sets invalid. # I therefore added these; they're a safeguard that results for specific descriptor types which is known to be faulty # is not included in the produced results spreadsheet # (in case they somehow slip the net) # Last known fault in code: unsigned integer subtraction in RICI comparison function # Date threshold corresponds to this commit def isQsiResultValid(fileCreationDateTime, resultJson): riciResultsValidAfter = datetime.datetime(year=2019, month=10, day=7, hour=15, minute=14, second=0, microsecond=0) return fileCreationDateTime > riciResultsValidAfter # Last known fault in code: override object count did not match # Date threshold corresponds to this commit def isSiResultValid(fileCreationDateTime, resultJson): siResultsValidAfter = datetime.datetime(year=2019, month=9, day=26, hour=17, minute=28, second=0, microsecond=0) hasCorrectOverrideObjectCount = 'overrideObjectCount' in resultJson and resultJson['overrideObjectCount'] == 10 hasCorrectSampleSetSize = resultJson['sampleSetSize'] == 10 hasValidCreationDateTime = siResultsValidAfter < fileCreationDateTime return hasCorrectOverrideObjectCount or hasCorrectSampleSetSize # Last known fault in code: none def is3dscResultValid(fileCreationTime, resultJson): return True def isQuicciResultValid(fileCreationTime, resultJson): return True def isFPFHResultValid(fileCreationTime, resultJson): return True # --- SETTINGS --- # Master input directories # Contains all data to be included in the spreadsheet # Format: [path to JSON output directory]: ([human readable name of result set], [cluster result set was executed on]) # Clusters used: # IDUN: Large cluster, though machines contain many different graphics cards. Must ABSOLUTELY NOT be used for time sensitive results # HEID: DGX-2 machine, contains 16 equivalent V100 cards. Used for time sensitive tests. inputDirectories = { # RICI, measurements for matching and computation times (only tested on 4 added clutter objects) '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_noearlyexit/output': ('QSI, No early exit, 5 objects', 'HEID'), '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_withearlyexit/output': ('QSI, Early exit, 5 objects', 'HEID'), # RICI, measurements for matching performance '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_idun_failed/output': ('Failed jobs from IDUN run', 'HEID'), '../input/results_computed_by_authors/IDUNRUNS/output_lotsofobjects_v4': ('primary QSI IDUN run', 'IDUN'), '../input/results_computed_by_authors/HEIDRUNS/output_seeds_qsi_v4_5objects_missing/output': ('re-run of 5 object QSI results that were missing raw files', 'HEID'), # SI, execution times and machine performance '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_10_objects_only/output': ('180 support angle, 10 objects', 'HEID'), '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_5_objects_only/output': ('180 support angle, 5 objects', 'HEID'), '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_180deg_si_missing/output': ('180 support angle, 10 objects', 'HEID'), # NOTE: 5 object sequence DOES NOT contribute to time measurements (the bit further down with lots of split() and merge() calls verifies this) '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_15': ('180 support angle, 1 & 5 objects', 'IDUN'), '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_10': ('180 support angle, 10 objects', 'IDUN'), '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_1': ('180 support angle, 1 object', 'IDUN'), # SI, testing matching performance of 60 degree support angle '../input/results_computed_by_authors/IDUNRUNS/output_smallsupportangle_lotsofobjects': ('60 support angle, primary', 'IDUN'), '../input/results_computed_by_authors/IDUNRUNS/output_qsifix_smallsupportangle_rerun': ('60 support angle, secondary', 'IDUN'), '../input/results_computed_by_authors/IDUNRUNS/output_supportanglechart60_si_v4_1': ('60 support angle, 1 object', 'IDUN'), '../input/results_computed_by_authors/IDUNRUNS/output_supportanglechart60_si_v4_5': ('60 support angle, 5 objects', 'IDUN'), '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_60deg_si_missing/output/': ('60 support angle, 10 objects', 'HEID'), '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_si_v4_60deg_5objects_missing/output/': ('60 support angle, 5 objects', 'HEID'), # 3DSC, execution time and matching performance are both measured '../input/results_computed_by_authors/HEIDRUNS/run1_3dsc_main/output/': ('3DSC', 'HEID'), # QUICCI, execution time and matching performance are both measured '../input/results_computed_by_authors/HEIDRUNS/run6_output_quicci_distance_functions_clutterResistant/output/': ('QUICCI', 'HEID'), # FPFH, execution time and matching performance are both measured '../input/results_computed_by_authors/HEIDRUNS/run9_output_fpfh_run/output/': ('FPFH', 'HEID'), } # The location where the master spreadsheet should be written to outfile = '../output/master_spreadsheet.xls' numCountsToIncludeInRawHistogramSpreadsheet = 100 fileMapLocation = '../output/filemap.json' with open('../res/seeds_used_to_create_charts.txt') as seedFile: seedsUsedToCreateCharts = [str(x).strip() for x in seedFile.readlines()] with open('../res/seeds_used_for_fpfh.txt') as seedFile: fpfhSeeds = [str(x).strip() for x in seedFile.readlines()] # Map of methods contained in the output files methods = { 'RICI': { 'isValid': isQsiResultValid, 'nameInJSONFile': 'qsi', 'namePrefixInJSONFile': 'QSI', 'generationTimings': ['total', 'meshScale', 'redistribution', 'generation'], 'searchTimings': ['total', 'search'] }, 'QUICCI': { 'isValid': isQuicciResultValid, 'nameInJSONFile': 'quicci', 'namePrefixInJSONFile': 'QUICCI', 'generationTimings': ['total', 'meshScale', 'redistribution', 'generation'], 'searchTimings': ['total', 'search'] }, 'SI': { 'isValid': isSiResultValid, 'nameInJSONFile': 'si', 'namePrefixInJSONFile': 'SI', 'generationTimings': ['total', 'initialisation', 'sampling', 'generation'], 'searchTimings': ['total', 'averaging', 'search'] }, '3DSC': { 'isValid': is3dscResultValid, 'nameInJSONFile': '3dsc', 'namePrefixInJSONFile': '3DSC', 'generationTimings': ['total', 'initialisation', 'sampling', 'pointCounting', 'generation'], 'searchTimings': ['total', 'search'] }, 'FPFH': { 'isValid': isFPFHResultValid, 'nameInJSONFile': 'fpfh', 'namePrefixInJSONFile': 'FPFH', 'generationTimings': ['total', 'reformat', 'spfh_origins', 'spfh_pointCloud', 'generation'], 'searchTimings': ['total', 'search'] } } # Settings for clutter heatmaps # Width and height of heatmap in pixels heatmapSize = 256 rawInputDirectories = { 'QSI': ['../input/results_computed_by_authors/HEIDRUNS/output_seeds_qsi_v4_5objects_missing/output/raw', '../input/results_computed_by_authors/IDUNRUNS/output_lotsofobjects_v4/raw'], 'QUICCI': ['../input/results_computed_by_authors/HEIDRUNS/run6_output_quicci_distance_functions_clutterResistant/output/raw'], 'SI': ['../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_5_objects_only/output/raw', '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_15/raw'], '3DSC': ['../input/results_computed_by_authors/HEIDRUNS/run1_3dsc_main/output/raw'], 'FPFH': ['../input/results_computed_by_authors/HEIDRUNS/run9_output_fpfh_run/output/raw'], } rawInputObjectCount = 5 clutterFileDirectories = ['../input/clutter_estimated_by_authors/clutter/'] # Distill the result set down to the entries we have all data for removeSeedsWithMissingEntries = True # Cut down the result set to a specific number of entries resultSetSizeLimit = 1500 enableResultSetSizeLimit = True # --- start of code --- print() print(' === Processing of experiment output files into the master spreadsheet ===') print('This spreadsheet contains the exact data used to construct the charts in the paper') print() matplotlib.use('Qt5Agg') # -- global initialisation -- # Maps seeds to a list of (dataset, value) tuples seedmap_top_result = {} seedmap_top10_results = {} for methodName in methods: seedmap_top_result[methodName] = {} seedmap_top10_results[methodName] = {} # -- code -- def extractExperimentSettings(loadedJson): settings = {} settings['boxSize'] = loadedJson['boxSize'] settings['sampleObjectCounts'] = loadedJson['sampleObjectCounts'] settings['sampleSetSize'] = loadedJson['sampleSetSize'] settings['searchResultCount'] = loadedJson['searchResultCount'] settings['spinImageSupportAngle'] = loadedJson['spinImageSupportAngle'] settings['spinImageWidth'] = loadedJson['spinImageWidth'] settings['spinImageWidthPixels'] = loadedJson['spinImageWidthPixels'] if 'overrideObjectCount' in loadedJson: settings['overrideObjectCount'] = loadedJson['overrideObjectCount'] else: settings['overrideObjectCount'] = max(loadedJson['sampleObjectCounts']) if 'descriptors' in loadedJson: settings['descriptors'] = loadedJson['descriptors'] else: descriptors = [] for method in methods: if methods[method]['namePrefixInJSONFile'] + 'histograms' in loadedJson: descriptors.append(methods[method]['nameInJSONFile']) settings['descriptors'] = descriptors settings['version'] = loadedJson['version'] return settings def loadOutputFileDirectory(path): originalFiles = os.listdir(path) # Filter out the raw output file directories originalFiles = [x for x in originalFiles if x != 'raw' and x != 'rawless'] results = { 'path': path, 'results': {}, 'settings': {}, 'fileOriginMap': {} } jsonCache = {} ignoredLists = {} allResultsInvalid = {} for method in methods: results['results'][methods[method]['namePrefixInJSONFile']] = {} ignoredLists[method] = [] allResultsInvalid[method] = False # Reading file contents for fileindex, file in enumerate(originalFiles): print(str(fileindex + 1) + '/' + str(len(originalFiles)), file + ' ', end='\r', flush=True) with open(os.path.join(path, file), 'r') as openFile: # Read JSON file try: fileContents = json.loads(openFile.read()) fileContents['resultSetOrigin'] = os.path.join(path, file) except Exception as e: print('FAILED TO READ FILE: ' + str(file)) print(e) continue jsonCache[file] = fileContents # Check validity of code by using knowledge about previous code changes filename_creation_time_part = file.split('_')[0] creation_time = datetime.datetime.strptime(filename_creation_time_part, "%Y-%m-%d %H-%M-%S") for method in methods: if not methods[method]['isValid'](creation_time, fileContents): ignoredLists[method].append(file) # If only a single result is deemed invalid, # THE ENTIRE SET is marked as invalid for this descriptor type # Data integrity is of the utmost importance, so I'm not taking any chances here. allResultsInvalid[method] = True # Processing file contents previousExperimentSettings = None for fileindex, file in enumerate(originalFiles): print(str(fileindex + 1) + '/' + str(len(originalFiles)), file + ' ', end='\r', flush=True) # Read JSON file fileContents = jsonCache[file] # Check if settings are the same as other files in the folder currentExperimentSettings = extractExperimentSettings(fileContents) if previousExperimentSettings is not None: if currentExperimentSettings != previousExperimentSettings: # Any discrepancy here is a fatal exception. It NEEDS attention regardless raise Exception("Experiment settings mismatch in the same batch! File: " + file) previousExperimentSettings = currentExperimentSettings results['settings'] = currentExperimentSettings results['fileOriginMap'][fileContents['seed']] = os.path.join(path, file) for method in methods: # Check for other incorrect settings. Ignore files if detected if 0 in fileContents['imageCounts']: if file not in ignoredLists[method]: ignoredLists[method].append(file) if fileContents['spinImageWidthPixels'] == 32: if file not in ignoredLists[method]: ignoredLists[method].append(file) # Save a lot of RAM by throwing away histogram data we don't need histogramsString = methods[method]['namePrefixInJSONFile'] + 'histograms' for sampleCountIndex, sampleObjectCount in enumerate(fileContents['sampleObjectCounts']): indexNameString = str(fileContents['sampleObjectCounts'][sampleCountIndex]) + ' objects' smallerHistogram = {} countThreshold = 12 if histogramsString in fileContents: if str(sampleCountIndex) in fileContents[histogramsString]: for x in range(0, countThreshold): if str(x) in fileContents[histogramsString][str(sampleCountIndex)]: smallerHistogram[str(x)] = fileContents[histogramsString][str(sampleCountIndex)][str(x)] fileContents[histogramsString][str(sampleCountIndex)] = smallerHistogram else: for x in range(0, countThreshold): if str(x) in fileContents[histogramsString][indexNameString]: smallerHistogram[str(x)] = fileContents[histogramsString][indexNameString][str(x)] fileContents[histogramsString][indexNameString] = smallerHistogram # Beauty checks if file not in ignoredLists[method] and allResultsInvalid[method]: ignoredLists[method].append(file) containsResultsForMethod = ('descriptors' in fileContents and methods[method]['nameInJSONFile'] in fileContents['descriptors']) or methods[method]['namePrefixInJSONFile'] + 'histograms' in fileContents # Sanity checks are done. We can now add any remaining valid entries to the result lists if not file in ignoredLists[method] and not allResultsInvalid[method] and containsResultsForMethod: results['results'][methods[method]['namePrefixInJSONFile']][str(fileContents['seed'])] = fileContents print() results['settings'] = previousExperimentSettings return results def objects(count): if count > 1: return 'Objects' else: return 'Object' print('Loading raw data files..') loadedRawResults = {} for method in methods: loadedRawResults[methods[method]['namePrefixInJSONFile']] = {} for algorithm in rawInputDirectories: for path in rawInputDirectories[algorithm]: print('Loading raw directory:', path) rawFiles = os.listdir(path) for fileIndex, file in enumerate(rawFiles): print(str(fileIndex + 1) + '/' + str(len(rawFiles)), file + ' ', end='\r', flush=True) seed = file.split('.')[0].split("_")[2] if not seed in loadedRawResults[algorithm]: with open(os.path.join(path, file), 'r') as openFile: # Read JSON file try: rawFileContents = json.loads(openFile.read()) loadedRawResults[algorithm][seed] = rawFileContents[algorithm][str(rawInputObjectCount)] except Exception as e: print('FAILED TO READ FILE: ' + str(file)) print(e) continue print() print() print('Loading input data files..') loadedResults = {} for directory in inputDirectories.keys(): print('Loading directory:', directory) loadedResults[directory] = loadOutputFileDirectory(directory) def filterResultSet(resultSet, index): out = copy.deepcopy(resultSet) for method in methods: sampleGenerationString = methods[method]['namePrefixInJSONFile'] + 'SampleGeneration' searchString = methods[method]['namePrefixInJSONFile'] + 'Search' histogramsString = methods[method]['namePrefixInJSONFile'] + 'histograms' if sampleGenerationString in out['runtimes']: for timingMeasurement in methods[method]['generationTimings']: out['runtimes'][sampleGenerationString][timingMeasurement] \ = [out['runtimes'][sampleGenerationString][timingMeasurement][index]] if searchString in out['runtimes']: for timingMeasurement in methods[method]['searchTimings']: out['runtimes'][searchString][timingMeasurement] = [out['runtimes'][searchString][timingMeasurement][index]] if histogramsString in out: # Older result dump files use an integer index as a way to specify which object count the results belong to # Newer dump files use the latter format, where it specifically lists the object count used # This if statement automatically switches between these variants if str(index) in out[histogramsString]: out[histogramsString] = {'0': out[histogramsString][str(index)]} else: out[histogramsString] = {'0': out[histogramsString][str(out['sampleObjectCounts'][index]) + ' objects']} return out def split(directory): print('Splitting', directory) global loadedResults result = loadedResults[directory] del loadedResults[directory] setMeta = inputDirectories[directory] del inputDirectories[directory] for itemCountIndex, itemCount in enumerate(result['settings']['sampleObjectCounts']): out = {'results': {}, 'settings': {}, 'fileOriginMap': {}} for method in methods: out['results'][methods[method]['namePrefixInJSONFile']] = {} out['settings'] = result['settings'].copy() out['settings']['sampleObjectCounts'] = [itemCount] out['fileOriginMap'] = result['fileOriginMap'].copy() for method in methods: for seed in result['results'][methods[method]['namePrefixInJSONFile']]: out['results'][methods[method]['namePrefixInJSONFile']][seed] = \ filterResultSet(result['results'][methods[method]['namePrefixInJSONFile']][seed], itemCountIndex) newDirectoryName = directory + ' (' + str(itemCount) + ' objects)' loadedResults[newDirectoryName] = out inputDirectories[newDirectoryName] = (setMeta[0] + ' (' + str(itemCount) + ' objects)', setMeta[1]) def merge(directory1, directory2, newdirectoryName, newDirectoryClusterName): global loadedResults global inputDirectories print('Merging', directory2, 'into', directory1) directory1_contents = loadedResults[directory1] directory2_contents = loadedResults[directory2] del loadedResults[directory1] del loadedResults[directory2] del inputDirectories[directory1] del inputDirectories[directory2] if directory1_contents['settings'] != directory2_contents['settings']: print( 'WARNING: Directories %s and %s have different generation settings, and may therefore not be compatible to be merged!' % ( directory1, directory2)) print('Directory 1:', directory1_contents['settings']) print('Directory 2:', directory2_contents['settings']) combinedResults = {'results': {}, 'settings': directory1_contents['settings'], 'fileOriginMap': directory1_contents['fileOriginMap']} for method in methods: combinedResults['results'][methods[method]['namePrefixInJSONFile']] = {} # Initialising with the original results for method in methods: combinedResults['results'][methods[method]['namePrefixInJSONFile']] = \ directory1_contents['results'][methods[method]['namePrefixInJSONFile']] additionCount = 0 # Now we merge any missing results into it for type in directory2_contents['results']: for seed in directory2_contents['results'][type].keys(): if seed not in combinedResults['results'][type]: combinedResults['results'][type][seed] = directory2_contents['results'][type][seed] additionCount += 1 for seed in directory2_contents['fileOriginMap'].keys(): if seed not in combinedResults['fileOriginMap']: combinedResults['fileOriginMap'][seed] = directory2_contents['fileOriginMap'][seed] loadedResults[newdirectoryName] = combinedResults inputDirectories[newdirectoryName] = (newdirectoryName, newDirectoryClusterName) print('\tAdded', additionCount, 'new values') return additionCount # Small hack, but silences a warning that does not apply here loadedResults['../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_idun_failed/output']['settings']['overrideObjectCount'] = 10 print('\nRestructuring datasets..\n') split('../input/results_computed_by_authors/IDUNRUNS/output_smallsupportangle_lotsofobjects') split('../input/results_computed_by_authors/IDUNRUNS/output_qsifix_smallsupportangle_rerun') split('../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_15') split('../input/results_computed_by_authors/IDUNRUNS/output_lotsofobjects_v4') split('../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_idun_failed/output') print() # QSI 1 object merge('../input/results_computed_by_authors/IDUNRUNS/output_lotsofobjects_v4 (1 objects)', '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_idun_failed/output (1 objects)', 'QSI, 1 object', 'HEID + IDUN') # QSI 5 objects merge('../input/results_computed_by_authors/HEIDRUNS/output_seeds_qsi_v4_5objects_missing/output', '../input/results_computed_by_authors/IDUNRUNS/output_lotsofobjects_v4 (5 objects)', 'QSI primary intermediate', 'HEID + IDUN') merge('QSI primary intermediate', '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_idun_failed/output (5 objects)', 'QSI, 5 objects', 'HEID + IDUN') # QSI 10 objects merge('../input/results_computed_by_authors/IDUNRUNS/output_lotsofobjects_v4 (10 objects)', '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_idun_failed/output (10 objects)', 'QSI, 10 objects', 'HEID + IDUN') # SI 180 degrees, 1 object merge('../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_1', '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_15 (1 objects)', 'SI 180 degrees, 1 object', 'IDUN') # SI 180 degrees, 5 objects additionCount = merge('../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_5_objects_only/output', '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_15 (5 objects)', 'SI 180 degrees, 5 objects', 'HEID') # this merge is mainly to remove the dataset from the input batch. We ultimately want the HEIDRUNS results exclusively because # we use these results to compare runtimes assert (additionCount == 0) # SI 180 degrees, 10 objects merge('../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_lotsofobjects_10_objects_only/output', '../input/results_computed_by_authors/IDUNRUNS/output_mainchart_si_v4_10', 'SI 180 degrees, 10 objects', 'HEID + IDUN') merge('SI 180 degrees, 10 objects', '../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_180deg_si_missing/output', 'SI 180 degrees, 10 objects', 'HEID + IDUN') # SI 60 degrees, 1 object merge('../input/results_computed_by_authors/IDUNRUNS/output_supportanglechart60_si_v4_1', '../input/results_computed_by_authors/IDUNRUNS/output_smallsupportangle_lotsofobjects (1 objects)', 'SI 60 degrees, 1 object intermediate', 'IDUN') merge('SI 60 degrees, 1 object intermediate', '../input/results_computed_by_authors/IDUNRUNS/output_qsifix_smallsupportangle_rerun (1 objects)', 'SI 60 degrees, 1 object', 'IDUN') # SI 60 degrees, 5 objects merge('../input/results_computed_by_authors/HEIDRUNS/output_qsifix_si_v4_60deg_5objects_missing/output/', '../input/results_computed_by_authors/IDUNRUNS/output_supportanglechart60_si_v4_5', 'SI 60 degrees, 5 objects, intermediate', 'HEID + IDUN') merge('SI 60 degrees, 5 objects, intermediate', '../input/results_computed_by_authors/IDUNRUNS/output_smallsupportangle_lotsofobjects (5 objects)', 'SI 60 degrees, 5 objects', 'HEID + IDUN') # SO 60 degrees, 10 objects merge('../input/results_computed_by_authors/HEIDRUNS/output_qsifix_v4_60deg_si_missing/output/', '../input/results_computed_by_authors/IDUNRUNS/output_smallsupportangle_lotsofobjects (10 objects)', 'SI 60 deg 10 objects intermediate', 'HEID + IDUN') merge('SI 60 deg 10 objects intermediate', '../input/results_computed_by_authors/IDUNRUNS/output_qsifix_smallsupportangle_rerun (10 objects)', 'SI 60 degrees, 10 objects', 'HEID + IDUN') print() print('\nProcessing..\n') seedSet = set() for directory in inputDirectories.keys(): for method in methods: for seed in loadedResults[directory]['results'][methods[method]['namePrefixInJSONFile']].keys(): seedSet.add(seed) seedList = [x for x in seedSet] print('Found', len(seedSet), 'seeds in result sets') if removeSeedsWithMissingEntries: print('\nRemoving missing entries..') missingSeeds = [] for directory in loadedResults: cutCount = 0 for seed in seedList: for method in methods: # Hack to allow FPFH to have missing entries if method == 'FPFH': continue if len(loadedResults[directory]['results'][methods[method]['namePrefixInJSONFile']]) > 0: if not seed in loadedResults[directory]['results'][methods[method]['namePrefixInJSONFile']]: missingSeeds.append(seed) cutCount += 1 print(directory, '- removed seed count:', cutCount) print('Detected', len(set(missingSeeds)), 'unique seeds with missing entries. Removing..') for missingSeed in missingSeeds: for directory in loadedResults: for method in methods: # FPFH seeds will be treated separately, so we don't remove them here if method == 'FPFH': continue if missingSeed in loadedResults[directory]['results'][methods[method]['namePrefixInJSONFile']]: del loadedResults[directory]['results'][methods[method]['namePrefixInJSONFile']][missingSeed] if missingSeed in seedList: del seedList[seedList.index(missingSeed)] print('\nLoading clutter files..') clutterFileMap = {} for clutterFileDirectory in clutterFileDirectories: print('Reading directory', clutterFileDirectory) clutterFiles = os.listdir(clutterFileDirectory) for clutterFileIndex, clutterFile in enumerate(clutterFiles): print(str(clutterFileIndex + 1) + '/' + str(len(clutterFiles)), clutterFile + ' ', end='\r', flush=True) with open(os.path.join(clutterFileDirectory, clutterFile), 'r') as openFile: # Read JSON file try: clutterFileContents = json.loads(openFile.read()) seed = clutterFileContents['sourceFile'].split('.')[0].split('_')[2] clutterFileMap[seed] = clutterFileContents except Exception as e: print('FAILED TO READ FILE: ' + str(file)) print(e) continue print() # Find the seeds for which all input sets have data print('\nComputing condensed seed set') rawSeedList = loadedRawResults[methods[[x for x in methods.keys()][0]]['namePrefixInJSONFile']].keys() print('Starting raw RICI seed set size:', len(rawSeedList)) for method in methods: # Hack to ensure all results are included (FPFH only has 500 results) if method == 'FPFH': continue rawSeedList = [x for x in rawSeedList if x in loadedRawResults[methods[method]['namePrefixInJSONFile']].keys()] print('Merged with ' + method + ':', len(rawSeedList)) #rawSeedList = [x for x in rawSeedList if x in seedList] #print('Merged with seedList:', len(rawSeedList)) rawSeedList = [x for x in rawSeedList if x in clutterFileMap.keys()] print('Merged with clutter file map:', len(rawSeedList)) rawSeedList = [x for x in rawSeedList if x in seedsUsedToCreateCharts] print('Merged with seed list used for paper:', len(rawSeedList)) print('Seeds remaining:', len(rawSeedList)) # We want to make sure our dataset contains entries for ALL data points seedList = rawSeedList if enableResultSetSizeLimit: seedList = [x for index, x in enumerate(seedList) if index < resultSetSizeLimit] rawSeedList = seedList print() print('Reduced result set to size', len(seedList)) print() print() print('Dumping file map..') correspondingFileMap = {} directoriesToDump = [ '../input/results_computed_by_authors/HEIDRUNS/run1_3dsc_main/output/', 'QSI, 1 object', 'QSI, 5 objects', 'QSI, 10 objects', 'SI 180 degrees, 1 object', 'SI 180 degrees, 5 objects', 'SI 180 degrees, 10 objects', 'SI 60 degrees, 1 object', 'SI 60 degrees, 5 objects', 'SI 60 degrees, 10 objects', '../input/results_computed_by_authors/HEIDRUNS/run6_output_quicci_distance_functions_clutterResistant/output/', '../input/results_computed_by_authors/HEIDRUNS/run9_output_fpfh_run/output/'] for directory in directoriesToDump: resultSet = loadedResults[directory] for method in methods: methodName = methods[method]['namePrefixInJSONFile'] if method == 'SI': # Need to distinguish between support angles if '60' in directory: method += '60' else: method += '180' if not method in correspondingFileMap: correspondingFileMap[method] = {} for seed in resultSet['results'][methodName].keys(): entry = resultSet['results'][methodName][seed] sourceFile = '' if int(seed) in resultSet['fileOriginMap']: sourceFile = resultSet['fileOriginMap'][int(seed)] for objectCount in entry['sampleObjectCounts']: if not str(objectCount) in correspondingFileMap[method]: correspondingFileMap[method][str(objectCount)] = {} correspondingFileMap[method][str(objectCount)][str(seed)] = sourceFile with open(fileMapLocation, 'w') as fileMapFile: fileMapFile.write(json.dumps(correspondingFileMap, indent=4)) print('Success.') print() # Create heatmap histograms histograms = {} for method in methods: histograms[method] = np.zeros(shape=(heatmapSize, heatmapSize), dtype=np.int64) print('Computing histograms..') histogramEntryCount = 0 for rawSeedIndex, rawSeed in enumerate(rawSeedList): print(str(rawSeedIndex + 1) + '/' + str(len(rawSeedList)) + " processed", end='\r', flush=True) clutterValues = clutterFileMap[rawSeed]['clutterValues'] histogramEntryCount += len(clutterValues) for method in methods: if not rawSeed in loadedRawResults[methods[method]['namePrefixInJSONFile']]: continue ranks = loadedRawResults[methods[method]['namePrefixInJSONFile']][rawSeed] if not(len(clutterValues) == len(ranks)): print('WARNING: batch size mismatch at seed', rawSeed , '!', 'method ' + method, [len(clutterValues), len(ranks)]) for i in range(0, len(clutterValues)): clutterValue = clutterValues[i] index = ranks[i] # Apparently some NaN in there if clutterValue is None: continue xBin = int((1.0 - clutterValue) * heatmapSize) if xBin >= heatmapSize: continue yBin = index if yBin < heatmapSize: histograms[method][heatmapSize - 1 - yBin, xBin] += 1 print('Histogram computed over', histogramEntryCount, 'values') for method in methods: histograms[method] = np.log10(np.maximum(histograms[method], 0.1)) extent = [0, heatmapSize, 0, heatmapSize] # Plot heatmap plt.clf() colorbar_ticks = np.arange(0, 8, 1) total_minimum_value = min([np.amin(histograms[x]) for x in histograms]) total_maximum_value = max([np.amax(histograms[x]) for x in histograms]) print('range:', total_minimum_value, total_maximum_value) #total_minimum_value = -1.0 #total_maximum_value = 7.168897592566977 normalisation = colors.Normalize(vmin=total_minimum_value,vmax=total_maximum_value) horizontal_ticks_real_coords = np.arange(0,256,25.599*2.0) horizontal_ticks_labels = [("%.1f" % x) for x in np.arange(0,1.1,0.2)] for figureIndex, method in enumerate(methods): plot = plt.figure(figureIndex + 1) plt.title(method.upper()) plt.ylabel('rank') plt.xlabel('clutter percentage') image = plt.imshow(histograms[method], extent=extent, cmap='nipy_spectral', norm=normalisation) plt.xticks(horizontal_ticks_real_coords, horizontal_ticks_labels) # Final chart gets the legend if figureIndex + 1 == len(methods.keys()): cbar = plt.colorbar(image, ticks=colorbar_ticks) cbar.ax.set_yticklabels(["{:.0E}".format(x) for x in np.power(10, colorbar_ticks)]) cbar.set_label('Sample count', rotation=90) plot.show() print() print('Heatmap generation complete, press enter to dump spreadsheets.') print('(which will also close the heatmap windows)') input() # -- Dump to spreadsheet -- #methods[method]['namePrefixInJSONFile'] print('Dumping spreadsheet..') book = xlwt.Workbook(encoding="utf-8") # Create data page for dataset settings table experimentSheet = book.add_sheet("Experiment Overview") allColumns = set() for directoryIndex, directory in enumerate(inputDirectories.keys()): result = loadedResults[directory] allColumns = allColumns.union(set(result['settings'])) # Overview table headers for keyIndex, key in enumerate(allColumns): experimentSheet.write(0, keyIndex + 1, str(key)) experimentSheet.write(0, len(allColumns) + 1, 'Cluster') for index, method in enumerate(methods): experimentSheet.write(0, len(allColumns) + index + 2, method + ' Count') # Overview table contents for directoryIndex, directory in enumerate(inputDirectories.keys()): directoryName, cluster = inputDirectories[directory] experimentSheet.write(directoryIndex + 1, 0, directoryName) result = loadedResults[directory] for keyIndex, key in enumerate(allColumns): if key in result['settings']: experimentSheet.write(directoryIndex + 1, keyIndex + 1, str(result['settings'][key])) else: experimentSheet.write(directoryIndex + 1, keyIndex + 1, ' ') experimentSheet.write(directoryIndex + 1, len(allColumns) + 1, cluster) for columnIndex, method in enumerate(methods): experimentSheet.write(directoryIndex + 1, len(allColumns) + columnIndex + 2, len([x for x in result['results'][methods[method]['namePrefixInJSONFile']] if x in seedList]) if method != 'FPFH' else len(fpfhSeeds) if len([x for x in result['results'][methods[method]['namePrefixInJSONFile']] if x in seedList]) != 0 else 0) # Sheets sheets = {} for method in methods: sheets[method] = {} for method in methods: sheets[method]['top0'] = book.add_sheet("Rank 0 " + method + " results") for method in methods: sheets[method]['top10'] = book.add_sheet("Top 10 " + method + " results") for method in methods: sheets[method]['generationSpeed'] = book.add_sheet(method + " Generation Times") for method in methods: sheets[method]['comparisonSpeed'] = book.add_sheet(method + " Comparison Times") vertexCountSheet = book.add_sheet("Reference Image Count") totalVertexCountSheet = book.add_sheet("Total Image Count") totalTriangleCountSheet = book.add_sheet("Total Triangle Count") # alter seedlist to also include additional FPFH ones baseSeedList = seedList seedList = seedList + [x for x in fpfhSeeds if not x in seedList] # Write initial columns for method in methods: sheets[method]['top0'].write(0, 0, 'seed') sheets[method]['top10'].write(0, 0, 'seed') sheets[method]['generationSpeed'].write(0, 0, 'seed') sheets[method]['comparisonSpeed'].write(0, 0, 'seed') vertexCountSheet.write(0, 0, 'seed') totalVertexCountSheet.write(0, 0, 'seed') totalTriangleCountSheet.write(0, 0, 'seed') for method in methods: for seedIndex, seed in enumerate(seedList): sheets[method]['top0'].write(seedIndex + 1, 0, seed) sheets[method]['top10'].write(seedIndex + 1, 0, seed) sheets[method]['generationSpeed'].write(seedIndex + 1, 0, seed) sheets[method]['comparisonSpeed'].write(seedIndex + 1, 0, seed) for seedIndex, seed in enumerate(seedList): vertexCountSheet.write(seedIndex + 1, 0, seed) totalVertexCountSheet.write(seedIndex + 1, 0, seed) totalTriangleCountSheet.write(seedIndex + 1, 0, seed) # seed column is 0, data starts at column 1 currentColumn = 1 for directoryIndex, directory in enumerate(inputDirectories.keys()): for methodIndex, method in enumerate(methods): resultSet = loadedResults[directory] directoryName, _ = inputDirectories[directory] sampleGenerationString = methods[method]['namePrefixInJSONFile'] + 'SampleGeneration' searchString = methods[method]['namePrefixInJSONFile'] + 'Search' histogramsString = methods[method]['namePrefixInJSONFile'] + 'histograms' # Writing column headers for sampleCountIndex, sampleObjectCount in enumerate(resultSet['settings']['sampleObjectCounts']): columnHeader = directoryName + ' (' + str(sampleObjectCount) + ' ' + objects( len(resultSet['settings']['sampleObjectCounts'])) + ')' sheets[method]['top0'].write(0, currentColumn + sampleCountIndex, columnHeader) sheets[method]['top10'].write(0, currentColumn + sampleCountIndex, columnHeader) sheets[method]['generationSpeed'].write(0, currentColumn + sampleCountIndex, columnHeader) sheets[method]['comparisonSpeed'].write(0, currentColumn + sampleCountIndex, columnHeader) if methodIndex == 0: vertexCountSheet.write(0, currentColumn + sampleCountIndex, columnHeader) totalVertexCountSheet.write(0, currentColumn + sampleCountIndex, columnHeader) totalTriangleCountSheet.write(0, currentColumn + sampleCountIndex, columnHeader) for seedIndex, seed in enumerate(seedList): # Write result into the dump file if: # - It exists in the method's set of results at all # - If it appears in the inclusion list of FPFH seeds, if it is an FPFH result # - If it appears in the inclusion list of regular seeds, if it is not an FPFH result if seed in resultSet['results'][methods[method]['namePrefixInJSONFile']] \ and (method != 'FPFH' or seed in fpfhSeeds)\ and (method == 'FPFH' or seed in baseSeedList): for sampleCountIndex, sampleObjectCount in enumerate(resultSet['settings']['sampleObjectCounts']): # Top 1 performance entry = resultSet['results'][methods[method]['namePrefixInJSONFile']][seed] totalImageCount = entry['imageCounts'][0] experimentIterationCount = len(resultSet['settings']['sampleObjectCounts']) percentageAtPlace0 = 0 totalImageCountInTop10 = 0 indexNameString = str(resultSet['settings']['sampleObjectCounts'][sampleCountIndex]) + ' objects' if str(sampleCountIndex) in entry[histogramsString]: if '0' in entry[histogramsString][str(sampleCountIndex)]: percentageAtPlace0 = float(entry[histogramsString][str(sampleCountIndex)]['0']) / float(totalImageCount) totalImageCountInTop10 = sum( [entry[histogramsString][str(sampleCountIndex)][str(x)] for x in range(0, 10) if str(x) in entry[histogramsString][str(sampleCountIndex)]]) else: if '0' in entry[histogramsString][indexNameString]: percentageAtPlace0 = float(entry[histogramsString][indexNameString]['0']) / float( totalImageCount) totalImageCountInTop10 = sum( [entry[histogramsString][indexNameString][str(x)] for x in range(0, 10) if str(x) in entry[histogramsString][indexNameString]]) sheets[method]['top0'].write(seedIndex + 1, currentColumn + sampleCountIndex, percentageAtPlace0) # Top 10 performance percentInTop10 = float(totalImageCountInTop10) / float(totalImageCount) sheets[method]['top10'].write(seedIndex + 1, currentColumn + sampleCountIndex, percentInTop10) # generation execution time generationTime = entry['runtimes'][sampleGenerationString]['total'][sampleCountIndex] sheets[method]['generationSpeed'].write(seedIndex + 1, currentColumn + sampleCountIndex, generationTime) # search execution time comparisonTime = entry['runtimes'][searchString]['total'][sampleCountIndex] sheets[method]['comparisonSpeed'].write(seedIndex + 1, currentColumn + sampleCountIndex, comparisonTime) # Vertex count sanity check vertexCountSheet.write(seedIndex + 1, currentColumn + sampleCountIndex, entry['imageCounts'][0]) totalVertexCountSheet.write(seedIndex + 1, currentColumn + sampleCountIndex, sum(entry['imageCounts'][0:sampleObjectCount])) totalTriangleCountSheet.write(seedIndex + 1, currentColumn + sampleCountIndex, sum(entry['vertexCounts'][0:sampleObjectCount]) / 3) else: for sampleCountIndex, sampleObjectCount in enumerate(resultSet['settings']['sampleObjectCounts']): sheets[method]['top0'].write(seedIndex + 1, currentColumn + sampleCountIndex, ' ') sheets[method]['top10'].write(seedIndex + 1, currentColumn + sampleCountIndex, ' ') sheets[method]['generationSpeed'].write(seedIndex + 1, currentColumn + sampleCountIndex, ' ') sheets[method]['comparisonSpeed'].write(seedIndex + 1, currentColumn + sampleCountIndex, ' ') # Moving on to the next column currentColumn += len(resultSet['settings']['sampleObjectCounts']) # beauty addition.. Cuts off final column for seedIndex, seed in enumerate(seedList + ['dummy entry for final row']): for method in methods: sheets[method]['top0'].write(seedIndex, currentColumn, ' ') sheets[method]['top10'].write(seedIndex, currentColumn, ' ') sheets[method]['generationSpeed'].write(seedIndex, currentColumn, ' ') sheets[method]['comparisonSpeed'].write(seedIndex, currentColumn, ' ') vertexCountSheet.write(seedIndex, currentColumn, ' ') totalVertexCountSheet.write(seedIndex, currentColumn, ' ') totalTriangleCountSheet.write(seedIndex, currentColumn, ' ') book.save(outfile) print('Complete.')