Omar branch

.gitignore

@@ -1,3 +1,7 @@
+# Data
+datasets/
+experiments/
+.idea/
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

requirements.txt

@@ -1,27 +1,27 @@
 # This file may be used to create an environment using:
 # $ conda create --name <env> --file <this file>
-cloudpickle=0.5.3
-cycler=0.10.0
-cython=0.28.3
-decorator=4.3.0
-h5py=2.10.0
-imageio=2.3.0
-kiwisolver=1.1.0
-matplotlib=3.1.1
-networkx=2.1
-ninja=1.9.0
-numpy=1.16
+cloudpickle==0.5.3
+cycler==0.10.0
+cython==0.28.3
+decorator==4.3.0
+h5py==2.10.0
+imageio==2.3.0
+kiwisolver==1.1.0
+matplotlib==3.1.1
+networkx==2.1
+ninja==1.9.0
+numpy==1.16
 pillow>=6.2.2
-pyqt=5.9.2
-pyyaml=5.1.2
-pywavelets=0.5.2
-qt=5.9.7
-scikit-image=0.15.0
-scikit-learn=0.21.3
-scipy=1.3.1
-six=1.12.0
-sqlite=3.29.0=h7b6447c_0
-toolz=0.9.0
-tqdm=4.31.1
-wheel=0.33.6
-yaml=0.1.7
+pyqt==5.9.2
+pyyaml==5.1.2
+pywavelets==0.5.2
+qt==5.9.7
+scikit-image==0.15.0
+scikit-learn==0.21.3
+scipy==1.3.1
+six==1.12.0
+sqlite==3.29.0
+toolz==0.9.0
+tqdm==4.31.1
+wheel==0.33.6
+yaml==0.1.7

scripts/SIMSG_gui.py

@@ -27,7 +27,6 @@
 
 import sys
 from builtins import enumerate
-
 import networkx as nx
 from grave import plot_network
 from grave.style import use_attributes
@@ -59,10 +58,10 @@
 plt.margins(0.0)
 
 parser = argparse.ArgumentParser()
-parser.add_argument('--checkpoint', default='./experiments/vg/spade_64_vg_model.pt')
+parser.add_argument('--checkpoint', default='E:/Fcis/4th Year Fcis/simsg/simsg/checkpoints/spade_64_vg_model.pt')
 parser.add_argument('--dataset', default='vg', choices=['clevr', 'vg'])
 parser.add_argument('--data_h5', default=None)
-parser.add_argument('--predgraphs', default=False, type=bool_flag)
+parser.add_argument('--predgraphs', default=True, type=bool_flag)
 parser.add_argument('--image_size', default=(64, 64), type=int_tuple)
 parser.add_argument('--num_samples', default=10000, type=int)
 parser.add_argument('--update_input', default=True, type=bool_flag)
@@ -79,7 +78,7 @@
     DATA_DIR = "./datasets/clevr/target/"
     args.data_image_dir = DATA_DIR
 else:
-    DATA_DIR = "./datasets/vg/"
+    DATA_DIR = "E:/Fcis/4th Year Fcis/simsg/simsg/datasets/vg/"
     args.data_image_dir = os.path.join(DATA_DIR, 'images')
 
 if args.data_h5 is None:
@@ -100,13 +99,13 @@
 
 def build_model():
     global checkpoint
-    checkpoint = torch.load(args.checkpoint)
+    checkpoint = torch.load(args.checkpoint, map_location=torch.device('cpu'))
 
     model = SIMSGModel(**checkpoint['model_kwargs'])
     model.load_state_dict(checkpoint['model_state'])
     model.eval()
     model.image_size = args.image_size
-    model.cuda()
+#   model.cuda()
     return model
 
 
@@ -465,9 +464,18 @@ def add_triple(self, is_subject=True):
         # extract user input
         anchor_label, anchor_idx = self.comboBox_obj2.currentText().split(".")
         new_pred = self.comboBox_p2.currentText()
-        pred_id = torch.tensor(vocab["pred_name_to_idx"][new_pred]).cuda()
+
+        if torch.cuda.is_available():
+            pred_id = torch.tensor(vocab["pred_name_to_idx"][new_pred]).cuda()
+        else:
+            pred_id = torch.tensor(vocab["pred_name_to_idx"][new_pred])
+
         new_node = self.comboBox_sub2.currentText()
-        new_node_idx = torch.tensor(vocab["object_name_to_idx"][new_node]).cuda()
+
+        if torch.cuda.is_available():
+            new_node_idx = torch.tensor(vocab["object_name_to_idx"][new_node]).cuda()
+        else:
+            new_node_idx = torch.tensor(vocab["object_name_to_idx"][new_node])
 
         anchor_idx = int(anchor_idx)
         imgbox_idx = self.objs.shape[0] - 1
@@ -484,7 +492,12 @@ def add_triple(self, is_subject=True):
         # add new box to list of boxes
         for box in self.boxes[:-1]:
             new_boxes.append(box)
-        new_boxes.append(torch.tensor([0, 0, 0, 0], dtype=torch.float32).cuda())
+
+        if torch.cuda.is_available():
+            new_boxes.append(torch.tensor([0, 0, 0, 0], dtype=torch.float32).cuda())
+        else:
+            new_boxes.append(torch.tensor([0, 0, 0, 0], dtype=torch.float32))
+
         new_boxes.append(self.boxes[-1])
 
         # expand and update the keep arrays. Set box and feat to 0 for the new node
@@ -692,8 +705,10 @@ def getfile(self):
         """
         self.batch = next(self.data_loader)
 
+        # self.imgs, self.objs, self.boxes, self.triples, self.obj_to_img, self.triple_to_img, self.imgs_in = \
+        #     [x.cuda() for x in self.batch]
         self.imgs, self.objs, self.boxes, self.triples, self.obj_to_img, self.triple_to_img, self.imgs_in = \
-            [x.cuda() for x in self.batch]
+                [x for x in self.batch]
 
         self.keep_box_idx = torch.ones_like(self.objs.unsqueeze(1), dtype=torch.float)
         self.keep_feat_idx = torch.ones_like(self.objs.unsqueeze(1), dtype=torch.float)
@@ -733,7 +748,7 @@ def gen_image(self):
                                query_feats=query_feats, keep_box_idx=self.keep_box_idx,
                                keep_feat_idx=self.keep_feat_idx, combine_gt_pred_box_idx=self.combine_gt_pred_box_idx,
                                keep_image_idx=self.keep_image_idx, random_feats=args.random_feats,
-                               get_layout_boxes=True)
+                               get_layout_boxes=False)
 
         imgs_pred, boxes_pred, masks_pred, noised_srcs, _, layout_boxes = model_out
 

scripts/evaluate_changes_vg.py

@@ -41,7 +41,7 @@
 parser.add_argument('--exp_dir', default='./experiments/vg/')
 parser.add_argument('--experiment', default="spade_vg", type=str)
 parser.add_argument('--checkpoint', default=None)
-parser.add_argument('--predgraphs', default=False, type=bool_flag)
+parser.add_argument('--predgraphs', default=True, type=bool_flag)
 parser.add_argument('--image_size', default=(64, 64), type=int_tuple)
 parser.add_argument('--shuffle', default=False, type=bool_flag)
 parser.add_argument('--loader_num_workers', default=0, type=int)
@@ -80,10 +80,18 @@
 if args.checkpoint is None:
     args.checkpoint = os.path.join(args.exp_dir, args.experiment + "_model.pt")
 
+def remove_vgg(model_state):
+  def filt(pair):
+    key, val = pair
+    return "high_level_feat" not in key
+
+  return dict(filter(filt, model_state.items()))
+
 
 def build_model(args, checkpoint):
   model = SIMSGModel(**checkpoint['model_kwargs'])
-  model.load_state_dict(checkpoint['model_state'])
+  new_state = remove_vgg(checkpoint['model_state'])
+  model.load_state_dict(new_state, strict=False)
   model.eval()
   model.image_size = args.image_size
   model.cuda()

scripts/evaluate_reconstruction.py

@@ -16,6 +16,8 @@
 
 import numpy as np
 import torch
+import time
+import datetime
 import os
 import yaml
 import tqdm
@@ -39,7 +41,7 @@
 
 from PerceptualSimilarity import models
 
-GPU = 1
+GPU = 0
 EVAL_ALL = True         # evaluate on all bounding boxes (batch size=1)
 IGNORE_SMALL = True
 
@@ -50,8 +52,8 @@
 parser.add_argument('--checkpoint', default=None)
 parser.add_argument('--dataset', default='vg', choices=['clevr', 'vg'])
 parser.add_argument('--with_feats', default=True, type=bool_flag)
-parser.add_argument('--generative', default=False, type=bool_flag)
-parser.add_argument('--predgraphs', default=False, type=bool_flag)
+parser.add_argument('--generative', default=True, type=bool_flag)
+parser.add_argument('--predgraphs', default=True, type=bool_flag)
 parser.add_argument('--mode', default='auto_nofeats', type=str)
 
 parser.add_argument('--data_h5', default=None)
@@ -84,9 +86,9 @@
 
 if args.checkpoint is None:
     ckpt = args.exp_dir + args.experiment
-    args.checkpoint = './{}_model.pt'.format(ckpt)
+    args.checkpoint = '{}_model.pt'.format(ckpt)
 
-CONFIG_FILE = args.exp_dir + 'logs/{}/args.yaml'.format(args.experiment)
+CONFIG_FILE = args.exp_dir + 'args_64_spade_vg.yaml'.format(args.experiment)
 IMG_SAVE_PATH = args.exp_dir + 'logs/{}/evaluation/'.format(args.experiment)
 RESULT_SAVE_PATH = args.exp_dir + 'logs/{}/evaluation/results/'.format(args.experiment)
 RESULT_FILE = RESULT_SAVE_PATH + '{}/test_results_{}.pickle'
@@ -96,7 +98,12 @@
 torch.cuda.set_device(GPU)
 device = torch.device(GPU)
 
+def remove_vgg(model_state):
+  def filt(pair):
+    key, val = pair
+    return "high_level_feat" not in key
 
+  return dict(filter(filt, model_state.items()))
 def main():
 
     if not os.path.isfile(args.checkpoint):
@@ -119,7 +126,8 @@ def main():
     # initialize model and load checkpoint
     kwargs = checkpoint['model_kwargs']
     model = SIMSGModel(**kwargs)
-    model.load_state_dict(checkpoint['model_state'])
+    new_state = remove_vgg(checkpoint['model_state'])
+    model.load_state_dict(new_state, strict=False)
     model.eval()
     model.to(device)
 

scripts/preprocess_vg.py

@@ -16,7 +16,7 @@
 
 import argparse, json, os
 from collections import Counter, defaultdict
-
+from tqdm import tqdm
 import numpy as np
 import h5py
 
@@ -314,11 +314,11 @@ def create_rel_vocab(args, image_ids, relationships, object_id_to_obj,
                      rel_aliases, vocab):
   pred_counter = defaultdict(int)
   image_ids_set = set(image_ids)
-  for image in relationships:
+  for image in tqdm(relationships):
     image_id = image['image_id']
     if image_id not in image_ids_set:
       continue
-    for rel in image['relationships']:
+    for rel in tqdm(image['relationships']):
       sid = rel['subject']['object_id']
       oid = rel['object']['object_id']
       found_subject = sid in object_id_to_obj

simsg/data/vg.py

@@ -42,7 +42,7 @@ def __init__(self, vocab, h5_path, image_dir, image_size=(256, 256),
 
     self.mode = mode
 
-    self.image_dir = image_dir
+    self.image_dir = bytes(image_dir, "utf-8")
     self.image_size = image_size
     self.vocab = vocab
     self.num_objects = len(vocab['object_idx_to_name'])

simsg/model.py

@@ -162,8 +162,11 @@ def __init__(self, vocab, image_size=(64, 64), embedding_dim=64,
 
     def build_obj_feats_net(self):
         # get VGG16 features for each object RoI
-        vgg_net = T.models.vgg16(pretrained=True)
-        layers = list(vgg_net.features._modules.values())[:-1]
+        #vgg_net = T.models.vgg16(pretrained=True)
+        #layers = list(vgg_net.features._modules.values())[:-1]
+
+        res_net = T.models.resnet50(pretrained=True)
+        layers = list(res_net.children())[:-2]
 
         img_feats = nn.Sequential(*layers)