diff --git a/.gitignore b/.gitignore index b6e4761..3f9e515 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Data +datasets/ +experiments/ +.idea/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/requirements.txt b/requirements.txt index 8bab878..92b87cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,27 @@ # This file may be used to create an environment using: # $ conda create --name --file -cloudpickle=0.5.3 -cycler=0.10.0 -cython=0.28.3 -decorator=4.3.0 -h5py=2.10.0 -imageio=2.3.0 -kiwisolver=1.1.0 -matplotlib=3.1.1 -networkx=2.1 -ninja=1.9.0 -numpy=1.16 +cloudpickle==0.5.3 +cycler==0.10.0 +cython==0.28.3 +decorator==4.3.0 +h5py==2.10.0 +imageio==2.3.0 +kiwisolver==1.1.0 +matplotlib==3.1.1 +networkx==2.1 +ninja==1.9.0 +numpy==1.16 pillow>=6.2.2 -pyqt=5.9.2 -pyyaml=5.1.2 -pywavelets=0.5.2 -qt=5.9.7 -scikit-image=0.15.0 -scikit-learn=0.21.3 -scipy=1.3.1 -six=1.12.0 -sqlite=3.29.0=h7b6447c_0 -toolz=0.9.0 -tqdm=4.31.1 -wheel=0.33.6 -yaml=0.1.7 +pyqt==5.9.2 +pyyaml==5.1.2 +pywavelets==0.5.2 +qt==5.9.7 +scikit-image==0.15.0 +scikit-learn==0.21.3 +scipy==1.3.1 +six==1.12.0 +sqlite==3.29.0 +toolz==0.9.0 +tqdm==4.31.1 +wheel==0.33.6 +yaml==0.1.7 diff --git a/scripts/SIMSG_gui.py b/scripts/SIMSG_gui.py index 6d5cab3..7847d67 100755 --- a/scripts/SIMSG_gui.py +++ b/scripts/SIMSG_gui.py @@ -27,7 +27,6 @@ import sys from builtins import enumerate - import networkx as nx from grave import plot_network from grave.style import use_attributes @@ -59,10 +58,10 @@ plt.margins(0.0) parser = argparse.ArgumentParser() -parser.add_argument('--checkpoint', default='./experiments/vg/spade_64_vg_model.pt') +parser.add_argument('--checkpoint', default='E:/Fcis/4th Year Fcis/simsg/simsg/checkpoints/spade_64_vg_model.pt') parser.add_argument('--dataset', default='vg', choices=['clevr', 'vg']) parser.add_argument('--data_h5', default=None) -parser.add_argument('--predgraphs', default=False, type=bool_flag) +parser.add_argument('--predgraphs', default=True, type=bool_flag) parser.add_argument('--image_size', default=(64, 64), type=int_tuple) parser.add_argument('--num_samples', default=10000, type=int) parser.add_argument('--update_input', default=True, type=bool_flag) @@ -79,7 +78,7 @@ DATA_DIR = "./datasets/clevr/target/" args.data_image_dir = DATA_DIR else: - DATA_DIR = "./datasets/vg/" + DATA_DIR = "E:/Fcis/4th Year Fcis/simsg/simsg/datasets/vg/" args.data_image_dir = os.path.join(DATA_DIR, 'images') if args.data_h5 is None: @@ -100,13 +99,13 @@ def build_model(): global checkpoint - checkpoint = torch.load(args.checkpoint) + checkpoint = torch.load(args.checkpoint, map_location=torch.device('cpu')) model = SIMSGModel(**checkpoint['model_kwargs']) model.load_state_dict(checkpoint['model_state']) model.eval() model.image_size = args.image_size - model.cuda() +# model.cuda() return model @@ -465,9 +464,18 @@ def add_triple(self, is_subject=True): # extract user input anchor_label, anchor_idx = self.comboBox_obj2.currentText().split(".") new_pred = self.comboBox_p2.currentText() - pred_id = torch.tensor(vocab["pred_name_to_idx"][new_pred]).cuda() + + if torch.cuda.is_available(): + pred_id = torch.tensor(vocab["pred_name_to_idx"][new_pred]).cuda() + else: + pred_id = torch.tensor(vocab["pred_name_to_idx"][new_pred]) + new_node = self.comboBox_sub2.currentText() - new_node_idx = torch.tensor(vocab["object_name_to_idx"][new_node]).cuda() + + if torch.cuda.is_available(): + new_node_idx = torch.tensor(vocab["object_name_to_idx"][new_node]).cuda() + else: + new_node_idx = torch.tensor(vocab["object_name_to_idx"][new_node]) anchor_idx = int(anchor_idx) imgbox_idx = self.objs.shape[0] - 1 @@ -484,7 +492,12 @@ def add_triple(self, is_subject=True): # add new box to list of boxes for box in self.boxes[:-1]: new_boxes.append(box) - new_boxes.append(torch.tensor([0, 0, 0, 0], dtype=torch.float32).cuda()) + + if torch.cuda.is_available(): + new_boxes.append(torch.tensor([0, 0, 0, 0], dtype=torch.float32).cuda()) + else: + new_boxes.append(torch.tensor([0, 0, 0, 0], dtype=torch.float32)) + new_boxes.append(self.boxes[-1]) # expand and update the keep arrays. Set box and feat to 0 for the new node @@ -692,8 +705,10 @@ def getfile(self): """ self.batch = next(self.data_loader) + # self.imgs, self.objs, self.boxes, self.triples, self.obj_to_img, self.triple_to_img, self.imgs_in = \ + # [x.cuda() for x in self.batch] self.imgs, self.objs, self.boxes, self.triples, self.obj_to_img, self.triple_to_img, self.imgs_in = \ - [x.cuda() for x in self.batch] + [x for x in self.batch] self.keep_box_idx = torch.ones_like(self.objs.unsqueeze(1), dtype=torch.float) self.keep_feat_idx = torch.ones_like(self.objs.unsqueeze(1), dtype=torch.float) @@ -733,7 +748,7 @@ def gen_image(self): query_feats=query_feats, keep_box_idx=self.keep_box_idx, keep_feat_idx=self.keep_feat_idx, combine_gt_pred_box_idx=self.combine_gt_pred_box_idx, keep_image_idx=self.keep_image_idx, random_feats=args.random_feats, - get_layout_boxes=True) + get_layout_boxes=False) imgs_pred, boxes_pred, masks_pred, noised_srcs, _, layout_boxes = model_out diff --git a/scripts/evaluate_changes_vg.py b/scripts/evaluate_changes_vg.py index 5acd554..0c73e7d 100755 --- a/scripts/evaluate_changes_vg.py +++ b/scripts/evaluate_changes_vg.py @@ -41,7 +41,7 @@ parser.add_argument('--exp_dir', default='./experiments/vg/') parser.add_argument('--experiment', default="spade_vg", type=str) parser.add_argument('--checkpoint', default=None) -parser.add_argument('--predgraphs', default=False, type=bool_flag) +parser.add_argument('--predgraphs', default=True, type=bool_flag) parser.add_argument('--image_size', default=(64, 64), type=int_tuple) parser.add_argument('--shuffle', default=False, type=bool_flag) parser.add_argument('--loader_num_workers', default=0, type=int) @@ -80,10 +80,18 @@ if args.checkpoint is None: args.checkpoint = os.path.join(args.exp_dir, args.experiment + "_model.pt") +def remove_vgg(model_state): + def filt(pair): + key, val = pair + return "high_level_feat" not in key + + return dict(filter(filt, model_state.items())) + def build_model(args, checkpoint): model = SIMSGModel(**checkpoint['model_kwargs']) - model.load_state_dict(checkpoint['model_state']) + new_state = remove_vgg(checkpoint['model_state']) + model.load_state_dict(new_state, strict=False) model.eval() model.image_size = args.image_size model.cuda() diff --git a/scripts/evaluate_reconstruction.py b/scripts/evaluate_reconstruction.py index ef9739e..f5599f3 100755 --- a/scripts/evaluate_reconstruction.py +++ b/scripts/evaluate_reconstruction.py @@ -16,6 +16,8 @@ import numpy as np import torch +import time +import datetime import os import yaml import tqdm @@ -39,7 +41,7 @@ from PerceptualSimilarity import models -GPU = 1 +GPU = 0 EVAL_ALL = True # evaluate on all bounding boxes (batch size=1) IGNORE_SMALL = True @@ -50,8 +52,8 @@ parser.add_argument('--checkpoint', default=None) parser.add_argument('--dataset', default='vg', choices=['clevr', 'vg']) parser.add_argument('--with_feats', default=True, type=bool_flag) -parser.add_argument('--generative', default=False, type=bool_flag) -parser.add_argument('--predgraphs', default=False, type=bool_flag) +parser.add_argument('--generative', default=True, type=bool_flag) +parser.add_argument('--predgraphs', default=True, type=bool_flag) parser.add_argument('--mode', default='auto_nofeats', type=str) parser.add_argument('--data_h5', default=None) @@ -84,9 +86,9 @@ if args.checkpoint is None: ckpt = args.exp_dir + args.experiment - args.checkpoint = './{}_model.pt'.format(ckpt) + args.checkpoint = '{}_model.pt'.format(ckpt) -CONFIG_FILE = args.exp_dir + 'logs/{}/args.yaml'.format(args.experiment) +CONFIG_FILE = args.exp_dir + 'args_64_spade_vg.yaml'.format(args.experiment) IMG_SAVE_PATH = args.exp_dir + 'logs/{}/evaluation/'.format(args.experiment) RESULT_SAVE_PATH = args.exp_dir + 'logs/{}/evaluation/results/'.format(args.experiment) RESULT_FILE = RESULT_SAVE_PATH + '{}/test_results_{}.pickle' @@ -96,7 +98,12 @@ torch.cuda.set_device(GPU) device = torch.device(GPU) +def remove_vgg(model_state): + def filt(pair): + key, val = pair + return "high_level_feat" not in key + return dict(filter(filt, model_state.items())) def main(): if not os.path.isfile(args.checkpoint): @@ -119,7 +126,8 @@ def main(): # initialize model and load checkpoint kwargs = checkpoint['model_kwargs'] model = SIMSGModel(**kwargs) - model.load_state_dict(checkpoint['model_state']) + new_state = remove_vgg(checkpoint['model_state']) + model.load_state_dict(new_state, strict=False) model.eval() model.to(device) diff --git a/scripts/preprocess_vg.py b/scripts/preprocess_vg.py index 97ebb7a..0e2eb1a 100644 --- a/scripts/preprocess_vg.py +++ b/scripts/preprocess_vg.py @@ -16,7 +16,7 @@ import argparse, json, os from collections import Counter, defaultdict - +from tqdm import tqdm import numpy as np import h5py @@ -314,11 +314,11 @@ def create_rel_vocab(args, image_ids, relationships, object_id_to_obj, rel_aliases, vocab): pred_counter = defaultdict(int) image_ids_set = set(image_ids) - for image in relationships: + for image in tqdm(relationships): image_id = image['image_id'] if image_id not in image_ids_set: continue - for rel in image['relationships']: + for rel in tqdm(image['relationships']): sid = rel['subject']['object_id'] oid = rel['object']['object_id'] found_subject = sid in object_id_to_obj diff --git a/simsg/data/vg.py b/simsg/data/vg.py index f8cd6bb..bacbbaf 100755 --- a/simsg/data/vg.py +++ b/simsg/data/vg.py @@ -42,7 +42,7 @@ def __init__(self, vocab, h5_path, image_dir, image_size=(256, 256), self.mode = mode - self.image_dir = image_dir + self.image_dir = bytes(image_dir, "utf-8") self.image_size = image_size self.vocab = vocab self.num_objects = len(vocab['object_idx_to_name']) diff --git a/simsg/model.py b/simsg/model.py index ad871e5..a958edc 100755 --- a/simsg/model.py +++ b/simsg/model.py @@ -162,8 +162,11 @@ def __init__(self, vocab, image_size=(64, 64), embedding_dim=64, def build_obj_feats_net(self): # get VGG16 features for each object RoI - vgg_net = T.models.vgg16(pretrained=True) - layers = list(vgg_net.features._modules.values())[:-1] + #vgg_net = T.models.vgg16(pretrained=True) + #layers = list(vgg_net.features._modules.values())[:-1] + + res_net = T.models.resnet50(pretrained=True) + layers = list(res_net.children())[:-2] img_feats = nn.Sequential(*layers)