1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
"""
Handles pre-rendered rooms generated by data.top_down
RenderedScene loads the pre-rendered data
along with other precomputed information,
and creates RenderedComposite, which combines those to 
create the multi-channel top-down view used in the pipeline
"""
from torch.utils import data
from data import ObjectCategories, House, Obj
import random
import numpy as np
import math
import pickle
import os
import json
import copy
import torch
import utils

class RenderedScene():
    """
    Loading a rendered room

    Attributes
    ----------
    category_map (ObjectCategories): object category mapping
        that should be the same across all instances of the class
    categories (list[string]): all categories present in this room type.
        Loaded once when the first room is loaded to reduce disk access.
    cat_to_index (dict[string, int]): maps a category to corresponding index
    current_data_dir (string): keep track of the current data directory, if 
        it changes, then categories and cat_to_index should be recomputed
    """
    category_map = ObjectCategories()
    categories = None
    cat_to_index = None
    current_data_dir = None

    def __init__(self, index, data_dir, data_root_dir=None, \
                 shuffle=True, load_objects=True, seed=None):
        """
        Load a rendered scene from file

        Parameters
        ----------
        index (int): room number
        data_dir (string): location of the pre-rendered rooms
        data_root_dir (string or None, optional): if specified,
            use this as the root directory
        shuffle (bool, optional): If true, randomly order the objects
            in the room. Otherwise use the default order as written
            in the original dataset
        load_objects (bool, optional): If false, only load the doors
            and windows. Otherwise load all objects in the room
        seed (int or None, optional): if set, use a fixed random seed
            so we can replicate a particular experiment
        """
        if seed:
            random.seed(seed)

        if not data_root_dir:
            data_root_dir = utils.get_data_root_dir()
        
        if RenderedScene.categories is None or RenderedScene.current_data_dir != data_dir:
            with open(f"{data_root_dir}/{data_dir}/final_categories_frequency", "r") as f:
                lines = f.readlines()
                cats = [line.split()[0] for line in lines]

            RenderedScene.categories = [cat for cat in cats if cat not in set(['window', 'door'])]
            RenderedScene.cat_to_index = {RenderedScene.categories[i]:i for i in range(len(RenderedScene.categories))}
            RenderedScene.current_data_dir = data_dir

        with open(f"{data_root_dir}/{data_dir}/{index}.pkl", "rb") as f:
            (self.floor, self.wall, nodes), self.room = pickle.load(f)

        self.object_nodes = []
        self.door_window_nodes = []
        for node in nodes:
            category = RenderedScene.category_map.get_final_category(node["modelId"])
            if category in ["door", "window"]:
                node["category"] = category
                self.door_window_nodes.append(node)
            elif load_objects:
                node["category"] = RenderedScene.cat_to_index[category]
                self.object_nodes.append(node)
        
        if shuffle:
            random.shuffle(self.object_nodes)
       
    def create_composite(self):
        """
        Create a initial composite that only contains the floor,
        wall, doors and windows. See RenderedComposite for how
        to add more objects
        """
        r = RenderedComposite(RenderedScene.categories, self.floor, self.wall, self.door_window_nodes)
        return r

class RenderedComposite():
    """
    Multi-channel top-down composite, used as input to NN
    """
    def __init__(self, categories, floor, wall, door_window_nodes=None):
        #Optional door_window just in case
        self.size = floor.shape[0]
        
        self.categories = categories
        
        self.room_mask = (floor + wall)
        self.room_mask[self.room_mask != 0] = 1
        
        self.wall_mask = wall.clone()
        self.wall_mask[self.wall_mask != 0] = 0.5
        
        self.height_map = torch.max(floor, wall)
        self.cat_map = torch.zeros((len(self.categories),self.size,self.size))

        self.sin_map = torch.zeros((self.size,self.size))
        self.cos_map = torch.zeros((self.size,self.size))

        self.door_map = torch.zeros((self.size, self.size))
        self.window_map = torch.zeros((self.size, self.size))

        if door_window_nodes:
            for node in door_window_nodes:
                h = node["height_map"]
                xsize, ysize = h.size()
                xmin = math.floor(node["bbox_min"][0])
                ymin = math.floor(node["bbox_min"][2])
                if xmin < 0: xmin = 0
                if ymin < 0: ymin = 0
                to_add = torch.zeros((self.size, self.size))
                to_add[xmin:xmin+xsize,ymin:ymin+ysize] = h
                update = to_add > self.height_map
                self.height_map[update] = to_add[update]
                self.wall_mask[to_add>0] = 1
                to_add[to_add>0] = 0.5
                if node["category"] == "door":
                    self.door_map = self.door_map + to_add
                else:
                    self.window_map = self.window_map + to_add
    
    def get_transformation(self, transform):
        """
        Bad naming, really just getting the sin and cos of the
        angle of rotation.
        """
        a = transform[0]
        b = transform[8]
        scale = (a**2+b**2)**0.5
        return (b/scale, a/scale)

    def add_height_map(self, to_add, category, sin, cos):
        """
        Add a new object to the composite. 
        Height map, category, and angle of rotation are
        all the information required.
        """
        update = to_add>self.height_map
        self.height_map[update] = to_add[update]
        mask = torch.zeros(to_add.size())
        mask[to_add>0] = 0.5
        self.cat_map[category] = self.cat_map[category] + mask
        self.sin_map[update] = (sin + 1) / 2
        self.cos_map[update] = (cos + 1) / 2

    def add_node(self, node):
        """
        Add a new object to the composite.
        Computes the necessary information and calls
        add_height_map
        """
        h = node["height_map"]
        category = node["category"]
        xsize, ysize = h.shape
        xmin = math.floor(node["bbox_min"][0])
        ymin = math.floor(node["bbox_min"][2])
        to_add = torch.zeros((self.size, self.size))
        to_add[xmin:xmin+xsize,ymin:ymin+ysize] = h
        sin, cos = self.get_transformation(node["transform"])
        self.add_height_map(to_add, category, sin, cos)

    def add_nodes(self, nodes):
        for node in nodes:
            self.add_node(node)
    
    def get_cat_map(self):
        return self.cat_map.clone()

    def add_and_get_composite(self, to_add, category, sin, cos, \
                              num_extra_channels=1, temporary=True):
        """
        Sometimes we need to create a composite to test
        if some objects should be added, without actually
        fixing the object to the scene. This method allows doing so.
        See get_composite.
        """
        if not temporary:
            raise NotImplementedError
        update = to_add>self.height_map
        mask = torch.zeros(to_add.size())
        mask[to_add>0] = 0.5
        composite = torch.zeros((len(self.categories)+num_extra_channels+8, self.size, self.size))
        composite[0] = self.room_mask
        composite[1] = self.wall_mask
        composite[2] = self.cat_map.sum(0) + mask
        composite[3] = self.height_map
        composite[3][update] = to_add[update]
        composite[4] = self.sin_map
        composite[4][update] = (sin + 1) / 2
        composite[5] = self.cos_map
        composite[5][update] = (cos + 1) / 2
        composite[6] = self.door_map
        composite[7] = self.window_map
        for i in range(len(self.categories)):
            composite[i+8] = self.cat_map[i]
        composite[8+category] += mask
        
        return composite


    def get_composite(self, num_extra_channels=1, ablation=None):
        """
        Create the actual multi-channel representation.
        Which is a N x img_size x img_size tensor.
        See the paper for more information.
        Current channel order:
            -0: room mask
            -1: wall mask
            -2: object mask
            -3: height map
            -4, 5: sin and cos of the angle of rotation
            -6, 7: single category channel for door and window
            -8~8+C: single category channel for all other categories

        Parameters
        ----------
        num_extra_channels (int, optional): number of extra empty 
            channels at the end. 1 for most tasks, 0 for should continue
        ablation (string or None, optional): if set, return a subset of all
            the channels for ablation study, see the paper for more details
        """
        if ablation is None:
            composite = torch.zeros((len(self.categories)+num_extra_channels+8, self.size, self.size))
            composite[0] = self.room_mask
            composite[1] = self.wall_mask
            composite[2] = self.cat_map.sum(0)
            composite[3] = self.height_map
            composite[4] = self.sin_map
            composite[5] = self.cos_map
            composite[6] = self.door_map
            composite[7] = self.window_map
            for i in range(len(self.categories)):
                composite[i+8] = self.cat_map[i]
        elif ablation == "depth":
            composite = torch.zeros((1+num_extra_channels, self.size, self.size))
            composite[0] = self.height_map
        elif ablation == "basic":
            composite = torch.zeros((6+num_extra_channels, self.size, self.size))
            composite[0] = self.room_mask
            composite[1] = self.wall_mask
            composite[2] = self.cat_map.sum(0)
            composite[3] = self.height_map
            composite[4] = self.sin_map
            composite[5] = self.cos_map
        else:
            raise NotImplementedError

        return composite