3

�L`�S�Q@s�
dZd
dl
mZ
ddlZddljZddlZddlZddl	j
jZddlj
Z
ddlZddlZddlZddlmZ
ddlZddlZddlZddlZejed�ZdZdZd	Zd
ZdZdZd
Z ej!ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddgP�
Z"dd�Z#Gdd�de$�Z%Gdd�de$�Z&Gdd�dej'�Z(dd�Z)dS)z�
Author: Changjian Chen
Date: 2020-11-27 10:23:23
LastEditTime: 2021-03-13 20:59:17
LastEditors: Changjian Chen
Description: 
FilePath: /WSL/CSD-SSD/data/coco17.py

�
)
�HOME�N)
�loggerzWSL/Data/coco17/�images�annotationszcocoapi/PythonAPIzshrink_instances_{}.jsonzcaptions_{}.jsonzlabel_extraction_{}.pklzshrink_coco_labels.txtTFc
Csbi}
t|d
�}g}xFt
|�
D]:\}}|jd�
}t|�
|
t|d�
<|j|djd�
�

qW|
|fS)N�
r�
,r��

)�open�	enumerate�split�int�append�strip)�
label_file�	label_map�labelsZclass_names�idx�line�ids�r�*/data/changjian/WSL/CSD-SSD/data/coco17.py�
get_label_mapDs









rc@s eZ
dZd
Zdd�Zdd�ZdS)�COCO17EvalAnnotationTransformz�Transforms a COCO annotation into a Tensor of bbox coords and label index
    Initilized with a dictionary lookup of classnames to indexes
    c

Cstt
jttt��
\|_|_dS)
N)r�osp�join�COCO17_ROOT�ANNOTATIONS�	LABEL_MAPr�classes)
�selfrrr�__init__Rs
z&COCO17EvalAnnotationTransform.__init__c
Cs�tj
||||g�
}g}x�|
D]�}d
|kr�|d
}i}|j|d}	|j|	|d<d|d<t|d�
dt|d�
dt|d|d�
dt|d|d�
dg|d
<|j|�

qtd	�

qW|S)
a

        Args:
            target (dict): COCO target json annotation as a python dict
            height (int): height
            width (int): width
        Returns:
            a list containing lists of bounding boxes  [bbox coords, class idx]
        �bbox�category_id�namerZ	difficultr
r	�zno bbox problem!)�np�arrayrr rr�print)
r!�target�width�height�scaleZobjects�objr#Z
obj_struct�	label_idxrrr�__call__Us 	












z&COCO17EvalAnnotationTransform.__call__N)�__name__�
__module__�__qualname__�__doc__r"r0rrrrrNs
rc@s eZ
dZd
Zdd�Zdd�ZdS)�COCO17AnnotationTransformz�Transforms a COCO annotation into a Tensor of bbox coords and label index
    Initilized with a dictionary lookup of classnames to indexes
    c
Cstt
jttt��
\|_}
dS)
N)rrrrrrr)r!�
_rrrr"{s
z"COCO17AnnotationTransform.__init__c
Cs�tj
||||g�
}g}x�|
D]�}d
|kr�|d
}|d|d7<|d|d7<|j|d}ttj
|�
|�
}	t|	dd�|	d<t|	dd�|	d<t|	dd�|	d<t|	dd�|	d<|	j|�

||	g
7}qtd	�

qW|S)
a

        Args:
            target (dict): COCO target json annotation as a python dict
            height (int): height
            width (int): width
        Returns:
            a list containing lists of bounding boxes  [bbox coords, class idx]
        r#r	rr&r
r$gg�?zno bbox problem!)r'r(r�list�max�minrr))
r!r*r+r,r-�resr.r#r/�	final_boxrrrr0~s"	















z"COCO17AnnotationTransform.__call__N)r1r2r3r4r"r0rrrrr5ws
r5c@s�eZ
dZd
Zedde�ddddddddfdd	�
Zd
d�Zdd
�Zdd�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zd d!�Zd"d#�Zd$d%�Zd&d'�Zd(d)�Zd*d+�Zd,d-�ZdS).�COCO17Detectiona�
`MS Coco Detection <http://mscoco.org/dataset/#detections-challenge2016>`_ Dataset.
    Args:
        root (string): Root directory where images are downloaded to.
        set_name (string): Name of the specific set of COCO images.
        transform (callable, optional): A function/transform that augments the
                                        raw images`
        target_transform (callable, optional): A function/transform that takes
        in the target (bbox) and transforms it.
    Z	train2017N�COCOg�?TFc
s�tj
jtj|
t��

d
dlm}

ttjt	t
t��
\�_�_
|�_tjttj|
t
d�d�j��
�_|�_tj|
t|��_|
tj|
t
tj|�
��
�_|
tj|
t
tj|�
��
�_tj
j|
t
d|d�}|jd�
d
k�
r tj|�
�
rtd�

t|d	�}|j�j d
�
j!d
�
}dd�|D�
�_"nt#d
�
�
ntd�

t$�jj%j&��
�_"|�_'|�_(|�_)|�_*|�_+|�
rv|�_,|t-�j"�
�_.nt/t-�j"�
|�
�_,|�_.�j0|	�

|jd�
d
k�
r��j,}n|}�j(�r(tj|
dj||j1dd����_2tjt�j2d�j��
�_3i}x(�j3D]}||t/|dd
j4��
<�qW|�_5�j'�r6tj|
dd�}t6j|dddddd��_7�j7j8dd�
�_9ttj
j|
d�d	��F}g�_:i�_;x2t<|�
D]&\}}|j d
�
�_:|�j;|j d
�
<�q�WWdQRXttj
j|
d�d��}t=j>|�
�_?WdQRXd}|t=j@jAd�j?jBd"�dd}t=jC�j?|gd
d��_D�jD�_D�j�r�tj|
dj|�
��_Etjt�jEd�j��
�_F�f
d d!��jD�
�_GdS)#Nr)
r=z
add_annos.pkl�rbZshrink_z_random_list.txt�trainzreading from random_listrr
c
Ssg|]}
t|
�
�qSr)
r)�.0�
irrr�
<listcomp>�sz,COCO17Detection.__init__.<locals>.<listcomp>z#you should have randomlist.txt filezreading ids from origin dataz@aggre_last/coco17_text_shrink_aggre_last_1_1e-5_{}/{}_result.pklZ2017�Zimage_idZbottom_up_attentionz(coco17_resnet101_faster_rcnn_genome.lmdbr
TF)�max_readers�readonly�lock�	readahead�meminit)
�writezcoco_open_vocab.txtzcoco_open_vocab_300d.npyg���Q��?r	)
�axisz,detections/processed_detections_train_{}.pklc
si|]}
|
�j|
�qSr)
r)r@�
k)
r!rr�
<dictcomp>	
sz,COCO17Detection.__init__.<locals>.<dictcomp>�����)H�sys�pathrrr�COCO_API�pycocotools.cocor=rrrrrr �	image_set�pickle�loadsr�read�	add_annos�finetune�IMAGES�root�
INSTANCES_SET�format�coco�CAPTIONS_SET�	coco_caps�os�count�existsr)rr
�_ids�
ValueErrorr7�	imgToAnns�keys�text�extracting_label�	transform�target_transformr%�
supervise_num�len�supervise_percentr�set_only_supervise�replace�extracted_file�_results�decode�results�lmdbZenv_bua�begin�txn_buaZ_open_vocabulary_list�
verb_2_idxrr'�loadZ_open_vocabulary_word_embedding�random�rand�shape�concatenate�embedding_array_dataZdetection_file�
detections�inverse_label_map)r!rYrRrhri�dataset_namerlrjrg�only_supervise�evalrfrWr=Zrandom_list_fileZrandom_listrZtrained_data_lenrrrZbua_feature_file�
frrZ
init_widthZoov_embr)
r!rr"�s�





















	




























 

 









zCOCO17Detection.__init__cCsF|
|_|
r,t
jd
�

|jd|j�j�|_nt
jd�

|jj�|_dS)Nzusing supervised onlyz&using both supervised and unsupervised)r�r�inforbrj�copyr)r!r�rrrrm
s






z"COCO17Detection.set_only_supervisec


Cs|jS)
N)
r )
r!rrrr
s
zCOCO17Detection.get_label_mapc


Cs|jS)
N)
rR)
r!rrr�get_set_type
szCOCO17Detection.get_set_typecCs|
rt�|_
dS)
N)rri)r!r�rrr�set_eval
s

zCOCO17Detection.set_evalcCs~
|jr`|j
|
�
}|j|
�
}|j|
�
}tj|jd
|jd
t|�
f�
jtj	�
}||t
j|�
|fS|j�
r<|j
|
�
\}}}}	}
}|jr�|j|
�
}n
|j
|
�
}|j|
�
\}
}|j
|
�
}||
jt�
||
jt�
<g}|d
d
kr�|}nNx*|D]"}|t|d�
d
kr�|j|�

q�Wt|�
d
k�
r.tjtj|�
|f�
}|||||fS|j|
�
\}}}	}
}|j�
rd|j|
�
}n
|j
|
�
}||||fSdS)z�
        Args:
            index (int): Index
        Returns:
            tuple: Tuple (image, target).
                   target is the object returned by ``coco.loadAnns``.
        rr
NrM)rf�pull_image_level_annos�pull_text_feature�pull_image_featurer'�onesrzrk�astype�float32�torch�
from_numpyrW�pull_detrg�pull_extracted_labels�pull_add_annos�boolrr�vstackr(�	pull_item)r!�index�labelZtext_featureZ
image_feature�masks�im�gt�det�
h�
w�semi�image_level_target�
text_add_anno�img_add_annoZ	gt_targetZnew_gt�
drrr�__getitem__ 
s:






























zCOCO17Detection.__getitem__c

Cs
t|j
�
S)
N)rkr)
r!rrr�__len__J
s
zCOCO17Detection.__len__cs�j|
}|
�j
kr"tjd
g
�
}ntjdg
�
}�jj|}�jj|d�
}tj�j|�
}||dd�d
fdk}�f
dd�|D�
}�jj|�
}|ddkr�g}t	|�
}t	|�
}	||}
t
j�j�jj
|�
dd�}t
j|�
s�td	j|�
�
�
tjt
j�j|��
}|j\}
}}�jdk	�
r&�j|
||
�}�jdk	�
r�tj|�
}�j||dd�dd
�f|dd�d
fd�\}}}}|dd�dd�df}tj|tj|d
d�f�
}|dk�
r�tj||	�
jt�
}|d|d|�j��}||d|�j�d�}tj|�
jddd
�|||
||fS)Nr
r)
�imgIdsr	g�������?c
s0g|](}
|
dd
�j��j
t|
d�
d��qS)N�r
)r#r$rM)�tolistr~r)r@r�)
r!rrrBW
s
z,COCO17Detection.pull_det.<locals>.<listcomp>�	file_namezImage path does not exist: {}r�T)
rJ�����)r	r
r)rrjr'r(r\rd�	getAnnIdsr}�loadAnnsrkrrrY�loadImgsra�AssertionErrorr[�cv2�imreadrzrirh�hstack�expand_dimsr�r�r��sumr�r��permute)r!r��img_idr�r*�ann_idsr�Znp_detZ
target_numZdet_numZorigin_combinedrO�imgr,r+r6�combined�boxesr�maskr)
r!rr�M
sB

































zCOCO17Detection.pull_detc
CsV
|j|
}|j
j|}|j
j|d
�
}|j
j|�
}tj|j|j
j|�
dd�}tj	|�
sft
dj|�
�
�
tj
tj|j|��
}|j\}}}	|jdk	r�|j|||�}|jdk	�
rtj|�
}|j||dd�dd�f|dd�df�\}}
}|dd�dd�d
f}tj|
tj|dd	�f�
}|
|jk�
r,tjdg
�
}ntjdg
�
}tj|�
jddd�||||fS)z�
        Args:
            index (int): Index
        Returns:
            tuple: Tuple (image, target, height, width).
                   target is the object returned by ``coco.loadAnns``.
        )
r�rr�zImage path does not exist: {}Nr�r	r
)
rJ)r	r
r)rr\rdr�r�rrrYr�rar�r[r�r�rzrirhr'r(r�r�rjr�r�r�)
r!r�r�r*r�rOr�r,r+r6r�rr�rrrr�u
s(
















zCOCO17Detection.pull_itemcCs6|j|
}|j
j|�
d
d}tjtj|j|�tj�S)a
Returns the original image object at index in PIL form

        Note: not using self.__getitem__(), as any transformations passed in
        could mess up this functionality.

        Argument:
            index (int): index of img to show
        Return:
            cv2 img
        rr�)	rr\r�r�r�rrrY�IMREAD_COLOR)r!r�r�rOrrr�
pull_image�
s


zCOCO17Detection.pull_imagecCs:|j|
}|j
j|d
�
}|j
j|�
}|j|dd�}||fS)a�
Returns the original annotation of image at index

        Note: not using self.__getitem__(), as any transformations passed in
        could mess up this functionality.

        Argument:
            index (int): index of img to get annotation of
        Return:
            list:  [img_id, [(label, bbox coords),...]]
                eg: ('001718', [('dog', (96, 13, 438, 332))])
        )
r�r
)rr\r�r�ri)r!r�r�r��annor�rrr�	pull_anno�
s





zCOCO17Detection.pull_annocCs�|j|
}|j
j|ggd
��}|d}|d}tjt|j�
�
}t|�
dkrXd|tj|�
<tjt|j�
�
}t|�
dkr�d|tj|�
<||fS)N)�imagerfr�rfrr
)rrV�getr'�zerosrkr r()r!r�r�Zadd_annor�r�Zimg_ohZtext_ohrrrr��
s











zCOCO17Detection.pull_add_annoscCsf|j|
}|j
j|d
�
}|j
j|�
}g}x8|D]0}|djd�
jd�
}dd�|D�
}|j|�

q.W|S)N)
r��caption�
.�
 c
Ssg|]}
|
j��qSr)
�lower)r@�
arrrrB�
sz-COCO17Detection.pull_text.<locals>.<listcomp>)rr^r�r�rr
�extend)r!r�r�r�Zannosrfr��caprrr�	pull_text�
s










zCOCO17Detection.pull_textcsL�j|
�
}�f
d
d�|D�
}t
jdd�|D�
�
}�j|}tj|jt
j�
�
S)Nc
sg|]}
�jj
|
d��qS)
�oov)rvr�)r@�
t)
r!rrrB�
sz5COCO17Detection.pull_text_feature.<locals>.<listcomp>c
Ssg|]}
|
dkr|
�qS)
r�r)r@r�rrrrB�
s)r�r'r(r|r�r�r�r�)r!r�rfZ	token_idsZ	token_embr)
r!rr��
s





z!COCO17Detection.pull_text_featurecCs2|j|
}|j
|}t|d
�
dk}|djt�
S)N�logitsg�?r)rrr�sigmoidr��float)r!r�r�r:�predrrrr��
s




z%COCO17Detection.pull_extracted_labelscCs|j|
}|j
|}|S)
N)rrr)r!r�r�r:rrr�
pull_metadata�
s




zCOCO17Detection.pull_metadatacCsp|j|
}t
j|jjt|�
j��
�
}t|d
�
}tj	t
j|d�
tjd�j
|d�}|dks^t�
tj|jtj�
�
S)N�	num_boxes�features)
�dtypei�$)rrSrTrur��str�encoderr'�
frombuffer�base64�	b64decoder��reshaper�r�r�r�)r!r�r�Zbua_metadatar�Zbua_featurerrrr��
s







z"COCO17Detection.pull_image_featurecCs:|j|
�
\}}d
}t
j|�
}x|D]}d||d<q"W|S)N�Ar
rM)r�r'r�)r!r�r�r�Zcat_numr�
grrrr��
s





z&COCO17Detection.pull_image_level_annosc
	Cs�d
|jj
d}
|
dj|j��
7}
|
dj|j�
7}
d}|
dj||jj�jdddt|�
��7}
d}|
d	j||j	j�jdddt|�
��7}
|
S)
NzDataset r
z    Number of datapoints: {}
z    Root Location: {}
z    Transforms (if any): z{0}{1}
r�z     Target Transforms (if any): z{0}{1})
�	__class__r1r[r�rYrh�__repr__rnrkri)r!�fmt_str�tmprrrr�s



*

*
zCOCO17Detection.__repr__)r1r2r3r4rr5r"rmrr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rrrrr<�s.	


a	*(#
	r<c
Csd
d
tj
|�
}
|
S)Nr
)r'�exp)�
x�
srrrr�s

r�)*r4�configrr_�os.pathrOrrNr��torch.utils.data�utils�dataZtorchvision.transforms�
transformsr��numpyr'rx�utils.loggerrrSrsr�r�rrrXrrPrZr]Z	LABEL_SETrr(Zselected_catr�objectrr5�Datasetr<r�rrrr�<module>	sJ






























)&u