Add evaluation code

Author: Shimingyi

Evaluation/README.md

@@ -0,0 +1,35 @@
+# Evaluation 
+
+## Data and pre-trained encoders
+
+We prepare all the comparison data with this [Onedrive](https://1drv.ms/u/s!AtagzSrg3_hppdxIvlkNf35Jxf-PBw?e=6cJV5H) link. You can download it and prepare this folder with following structure:
+
+```
+data\
+---- exp1_mann+dp
+---- exp1_mann+lp
+---- exp1_moglow
+---- exp1_ours
+...
+feature_extractor\
+---- 100style_position_distribution.npz
+---- 100style_position_classifier.pth
+...
+utils\
+...
+result_recording\
+...
+```
+
+## Commands
+
+Then you can run our evaluation with following commands:
+``` shell
+# Produce the deep latent for each motion data
+python compute_feat.py
+
+# Compute the metrics
+python eval_motion_FID.py
+python eval_motion_quality.py
+python eval_traj_alignment.py
+```

Evaluation/compute_feat.py

@@ -0,0 +1,67 @@
+import os
+import torch
+from tqdm import tqdm
+import numpy as np
+from feature_extractor.train_classifier import Classifier, window_size
+from feature_extractor.process_train_data import styleList, process_file
+import multiprocessing
+from multiprocessing import Pool
+
+def make_data(bvh_paths):
+    with Pool(multiprocessing.cpu_count()) as p:
+        data_list = list(tqdm(p.imap(process_file, bvh_paths), total=len(bvh_paths)))
+    data_list = [data for data in data_list if data is not None]
+    return data_list
+
+
+if __name__ == '__main__':
+    
+    clip_len = window_size
+    
+    process_folder = [
+        ["data/exp1_mann+dp", "ybot_skeleton"],
+        ["data/exp1_mann+lp", "ybot_skeleton"],
+        ["data/exp1_moglow", "style_skeleton"],
+        ["data/exp1_ours", "style_skeleton"],
+        ["data/exp1_ours_mlp", "style_skeleton"],
+        ["data/exp2_mann+dp", "ybot_skeleton"],
+        ["data/exp2_mann+lp", "ybot_skeleton"],
+        ["data/exp2_moglow", "style_skeleton"],
+        ["data/exp2_ours", "style_skeleton"],
+    ]
+    
+    bvh_paths = []
+    for data_folder, skeleton_config in process_folder:
+        for root, dirs, files in os.walk(data_folder):
+            for file in files:
+                if '.bvh' in file:
+                    bvh_paths.append([os.path.join(root, file), skeleton_config])
+    
+    motion_data_list = make_data(bvh_paths)
+    local_positions = [data['local_position'] for data in motion_data_list]
+    
+    device = torch.device('cpu')
+    classifier = Classifier(in_feat=45, class_num=100, device=device)
+    save_dict = torch.load('feature_extractor/100style_position_classifier.pth', map_location=device)
+    classifier.load_state_dict(save_dict['state_dict'])
+    classifier.eval()
+    style_names = save_dict['class_names']
+    
+    for data in motion_data_list:
+        save_path = data['file_path'].replace('.bvh', '_predicted.npz')
+        if os.path.exists(save_path):
+            continue
+        local_position = data['local_position']
+        frame_num = local_position.shape[0]
+        local_position = local_position.reshape(frame_num, -1)
+        clip_indices = np.arange(0, frame_num - clip_len + 1, clip_len)[:, None] + np.arange(clip_len)
+        model_input = local_position[clip_indices]
+        model_input = torch.tensor(model_input, dtype=torch.float32).to(device)
+        with torch.no_grad():
+            feats, out = classifier(model_input.permute(0, 2, 1))
+            _, example_predicted = torch.max(out.data, 1)
+            pre_style_name = [style_names[idx] for idx in example_predicted.cpu().numpy()]
+        
+        feats_numpy = feats.cpu().numpy()
+        np.savez(save_path, feats=feats_numpy, predicted=pre_style_name)
+        print('Processed:', save_path)

Evaluation/eval_motion_FID.py

@@ -0,0 +1,88 @@
+import os
+import numpy as np
+from tqdm import tqdm
+import scipy
+
+def calculate_metrics(args):
+    bvh_path, gt_distribution, save_list = args
+    feature_path = bvh_path.replace('.bvh', '_predicted.npz')
+    feature = np.load(feature_path)['feats']
+    if 'exp1' in bvh_path or 'diffusion_step' in bvh_path:
+        style_name = bvh_path.split('/')[-1].split('_')[-1].split('.')[0]
+        gt_mu, gt_sigma = gt_distribution[style_name]
+        mu, sigma = np.mean(feature, axis=0), np.cov(feature, rowvar=False)
+        ssd = np.sum((mu - gt_mu)**2.0)
+        covmean = scipy.linalg.sqrtm(sigma.dot(gt_sigma))
+        
+        if np.iscomplexobj(covmean):
+            covmean = covmean.real
+        
+        fid = ssd + np.trace(sigma + gt_sigma - 2.0 * covmean)
+        save_list.append(fid/len(sigma))
+    if 'exp2' in bvh_path:
+        start_style_name = bvh_path.split('/')[-1].split('.')[0].split('_')[1]
+        gt_mu, gt_sigma = gt_distribution[start_style_name]
+        start_feature = feature[:len(feature)//2]
+        mu, sigma = np.mean(start_feature, axis=0), np.cov(start_feature, rowvar=False)
+        ssd = np.sum((mu - gt_mu)**2.0)
+        covmean = scipy.linalg.sqrtm(sigma.dot(gt_sigma))
+        if np.iscomplexobj(covmean):
+            covmean = covmean.real
+        start_fid = ssd + np.trace(sigma + gt_sigma - 2.0 * covmean)
+        
+        target_style_name = bvh_path.split('/')[-1].split('.')[0].split('_')[3]
+        gt_mu, gt_sigma = gt_distribution[target_style_name]
+        target_feature = feature[len(feature)//2:]
+        mu, sigma = np.mean(target_feature, axis=0), np.cov(target_feature, rowvar=False)
+        ssd = np.sum((mu - gt_mu)**2.0)
+        covmean = scipy.linalg.sqrtm(sigma.dot(gt_sigma))
+        if np.iscomplexobj(covmean):
+            covmean = covmean.real
+        target_fid = ssd + np.trace(sigma + gt_sigma - 2.0 * covmean)
+        save_list.append((start_fid + target_fid)/2/len(sigma))
+    # print('Finish %s' % bvh_path)
+
+if __name__ == '__main__':
+
+    test_folders = [
+        "data/exp1_mann+dp",
+        "data/exp1_mann+lp",
+        "data/exp1_moglow",
+        "data/exp1_ours",
+        "data/exp1_ours_mlp",
+        "data/exp2_mann+dp",
+        "data/exp2_mann+lp",
+        "data/exp2_moglow",
+        "data/exp2_ours"
+    ]
+
+    result_path = os.path.join('./result_recording', 'motion_FID.txt')
+    gt_distribution = np.load('feature_extractor/100style_position_distribution.npz', allow_pickle=True)['style_features_dic'].item()
+
+    if not os.path.exists(result_path):
+        with open(result_path, 'w') as f:
+            f.write('Metrics: \t\t FID\t\n')
+            f.write('---------------------------------\n')
+
+    for test_folder in test_folders:
+        test_file_list = [os.path.join(test_folder, f) for f in os.listdir(test_folder) if f.endswith('.bvh')]
+        calculate_metrics((test_file_list[0], gt_distribution, []))
+
+        metric_list = []
+        args = [(test_file, gt_distribution, metric_list) for test_file in test_file_list]
+        for arg in tqdm(args):
+            calculate_metrics(arg)
+        
+        
+        # num_processes = multiprocessing.cpu_count()
+        # with multiprocessing.Pool(processes=num_processes) as pool:
+        #     pool.map(calculate_metrics, args)
+        
+        # metric_list = list(metric_list)
+        avg_value = np.mean(metric_list)
+        
+        with open(result_path, 'a') as f:
+            f.write(test_folder + '\t' + '%.4f\t\n' % avg_value)
+            f.write('\n')
+
+        print('Finish %s, metrics: %s' % (test_folder, avg_value))

Evaluation/eval_motion_quality.py

@@ -0,0 +1,85 @@
+import os
+import numpy as np
+from tqdm import tqdm
+import utils.motion_modules as motion_modules
+from utils.bvh_motion import Motion
+import multiprocessing
+
+
+def calculate_metrics(args):
+    bvh_path, metric_names, save_dic = args
+    motion = Motion.load_bvh(bvh_path)
+    
+    foot_idx = []
+    if 'RightToeBase' in motion.names:
+        foot_idx.append(motion.names.index('LeftToeBase'))
+        foot_idx.append(motion.names.index('RightToeBase'))
+    if 'RightToe' in motion.names:
+        foot_idx.append(motion.names.index('LeftToe'))
+        foot_idx.append(motion.names.index('RightToe'))    
+    motion = motion_modules.on_ground(motion, foot_idx)
+    metric_value = []
+    for metric_name in metric_names:
+        if metric_name == 'foot_sliding':
+            metric_value.append(motion_modules.cal_metric_foot_sliding(motion, y_threshold=1)) # cm
+        elif metric_name == 'cohenrence':
+            metric_value.append(motion_modules.cal_metric_cohenrence(motion))
+    save_dic[bvh_path] = metric_value
+    
+
+if __name__ == '__main__':
+
+    test_folders = [
+        "data/exp1_mann+dp",
+        "data/exp1_mann+lp",
+        "data/exp1_moglow",
+        "data/exp1_ours",
+        "data/exp1_ours_mlp",
+        "data/exp2_mann+dp",
+        "data/exp2_mann+lp",
+        "data/exp2_moglow",
+        "data/exp2_ours",
+    ]
+
+    select_metric = ['foot_sliding', 'cohenrence']
+    result_path = os.path.join('./result_recording', 'motion_quality.txt')
+
+    if not os.path.exists(result_path):
+        with open(result_path, 'w') as f:
+            f.write('Metrics: \t\t')
+            for metric_name in select_metric:
+                f.write('%s\t\t' % metric_name)
+            f.write('\n')
+            f.write('---------------------------------\n')
+
+    for test_folder in test_folders:
+        
+        test_file_list = [os.path.join(test_folder, f) for f in os.listdir(test_folder) if f.endswith('.bvh')]
+
+        calculate_metrics((test_file_list[0], select_metric, {}))
+
+        metric_dic = multiprocessing.Manager().dict()
+        args = [(test_file, select_metric, metric_dic) for test_file in test_file_list]
+        
+        num_processes = multiprocessing.cpu_count()
+        with multiprocessing.Pool(processes=num_processes) as pool:
+            pool.map(calculate_metrics, args)
+        
+        metric_dic = dict(metric_dic)
+
+        avg_list = []
+        for metric_name in select_metric:
+            metric_value = []
+            for test_file in test_file_list:
+                metric_value.append(metric_dic[test_file][select_metric.index(metric_name)])
+            # remove nan in metric_value
+            metric_value = [value for value in metric_value if not np.isnan(value)]
+            avg_list.append(np.mean(metric_value))
+        
+        with open(result_path, 'a') as f:
+            f.write(test_folder + '\t')            
+            for metric_name in select_metric:
+                f.write('%.4f\t\t' % avg_list[select_metric.index(metric_name)])
+            f.write('\n')
+
+        print('Finish %s, metrics: %s' % (test_folder, avg_list))

Evaluation/eval_traj_alignment.py

@@ -0,0 +1,112 @@
+import os
+import numpy as np
+from tqdm import tqdm
+import utils.motion_modules as motion_modules
+from utils.bvh_motion import Motion
+import multiprocessing
+
+
+def calculate_metrics(args):
+    bvh_path, preset_path, metric_names, save_dic = args
+    motion = Motion.load_bvh(bvh_path)
+    
+    all_frame_idx = np.arange(motion.frame_num)
+    if 'LeftCollar' in motion.names:
+        forward_angle = motion_modules.extract_forward(motion, all_frame_idx, 'LeftCollar', 'RightCollar', 'LeftHip', 'RightHip')
+    elif 'RightUpLeg' in motion.names:
+        forward_angle = motion_modules.extract_forward(motion, all_frame_idx, 'LeftShoulder', 'RightShoulder', 'LeftUpLeg', 'RightUpLeg')
+
+    forward_angle = np.rad2deg(forward_angle)
+    preset_traj, preset_orien = [], []
+    with open(preset_path, 'r') as f:
+        for line in f:
+            values = line.strip().split(',')
+            traj_x, traj_z = float(values[0]), float(values[2])
+            traj_angle = np.arctan2(traj_z, traj_x)
+            dirc_x, dirc_z = float(values[3]), float(values[5])
+            dirc_angle = np.arctan2(dirc_z, -dirc_x)
+            preset_traj.append(traj_angle)
+            preset_orien.append(dirc_angle)
+    preset_traj, preset_orien = np.rad2deg(np.array(preset_traj)), np.rad2deg(np.array(preset_orien))
+    
+    if motion.frame_num > len(preset_traj):
+        motion = motion_modules.temporal_scale(motion, 2)
+        forward_angle = forward_angle[::2]
+    
+    preset_traj = preset_traj[:motion.frame_num-1]
+    preset_orien = preset_orien[:motion.frame_num]
+    
+    traj_pos = motion.positions[:, 0, [0, 2]]
+    traj_pos[:, 0] *= -1
+    traj_pos_diff = np.diff(traj_pos, axis=0)
+    traj_angle = np.rad2deg(np.arctan2(traj_pos_diff[:, 1], traj_pos_diff[:, 0])) 
+    
+    metric_value = []
+    for metric_name in metric_names:
+        if metric_name == 'traj_error':
+            value = np.abs(np.mean(preset_traj - traj_angle))
+            # value = np.min([value, 90 - value, 180 - value])
+            metric_value.append(np.abs(value))
+        elif metric_name == 'orien_error':
+            value = np.abs(np.mean(preset_orien - forward_angle))
+            # value = np.min([value, 90 - value, 180 - value])
+            metric_value.append(np.abs(value))
+    
+    save_dic[bvh_path] = metric_value
+    
+
+if __name__ == '__main__':
+
+    test_folders = [
+            ["data/exp1_mann+dp", "data/preset.txt"],
+            ["data/exp1_mann+lp", "data/preset.txt"],
+            ["data/exp1_matching", "data/preset.txt"],
+            ["data/exp1_moglow", "data/preset.txt"],
+            ["data/exp1_ours", "data/preset.txt"],
+            ["data/exp1_ours_mlp", "data/preset.txt"],
+        ]
+
+    select_metric = ['traj_error', 'orien_error']
+    result_path = os.path.join('./result_recording', 'trajectory_alignment.txt')
+
+    if not os.path.exists(result_path):
+        with open(result_path, 'w') as f:
+            f.write('Metrics: \t\t')
+            for metric_name in select_metric:
+                f.write('%s\t\t' % metric_name)
+            f.write('\n')
+            f.write('---------------------------------\n')
+
+    for test_folder, preset_path in test_folders:
+        test_file_list = [os.path.join(test_folder, f) for f in os.listdir(test_folder) if f.endswith('.bvh')]
+
+        calculate_metrics((test_file_list[0], preset_path, select_metric, {}))
+
+        metric_dic = multiprocessing.Manager().dict()
+        args = [(test_file, preset_path, select_metric, metric_dic) for test_file in test_file_list]
+        
+        num_processes = multiprocessing.cpu_count()
+        with multiprocessing.Pool(processes=num_processes) as pool:
+            pool.map(calculate_metrics, args)
+        
+        metric_dic = dict(metric_dic)
+
+        avg_list = []
+        for metric_name in select_metric:
+            metric_value = []
+            for test_file in test_file_list:
+                metric_value.append(metric_dic[test_file][select_metric.index(metric_name)])
+            # remove nan in metric_value
+            metric_value = [value for value in metric_value if not np.isnan(value)]
+            avg_list.append(np.mean(metric_value))
+        
+        with open(result_path, 'a') as f:
+            f.write(test_folder + '\t')            
+            for metric_name in select_metric:
+                f.write('%.4f\t\t' % avg_list[select_metric.index(metric_name)])
+            f.write('\n')
+
+        print('Finish %s, metrics: %s' % (test_folder, avg_list))
+            
+            
+