diff --git a/prime/README.md b/prime/README.md new file mode 100644 index 00000000000..efe18496a7d --- /dev/null +++ b/prime/README.md @@ -0,0 +1,3 @@ +## Relocated project + +PRIME has been moved to [its own repository](https://github.com/cctbx-xfel/prime). diff --git a/prime/SConscript b/prime/SConscript deleted file mode 100644 index b1cd92432c5..00000000000 --- a/prime/SConscript +++ /dev/null @@ -1,39 +0,0 @@ -import libtbx.load_env -import os -Import("env_base", "env_etc") - -env_etc.prime_dist = libtbx.env.dist_path("prime") -env_etc.prime_include = os.path.dirname(env_etc.prime_dist) -env_etc.prime_common_includes = [ - env_etc.prime_include, - env_etc.libtbx_include, - env_etc.cctbx_include, - env_etc.scitbx_include, - env_etc.chiltbx_include, - env_etc.omptbx_include, - env_etc.boost_include, -] - -env = env_base.Clone(SHLINKFLAGS=env_etc.shlinkflags) -env.Append(LIBS=["cctbx"] + env_etc.libm) -env_etc.include_registry.append( - env=env, - paths=env_etc.prime_common_includes) -if (env_etc.static_libraries): builder = env.StaticLibrary -else: builder = env.SharedLibrary - -if (not env_etc.no_boost_python): - Import("env_boost_python_ext") - env_prime_boost_python_ext = env_boost_python_ext.Clone() - env_prime_boost_python_ext.Prepend( - LIBS=["cctbx", "scitbx_boost_python"]) - env_prime_boost_python_ext.SharedLibrary( - target="#lib/prime_ext", source="ext.cpp") - - - env_etc.include_registry.append( - env=env_prime_boost_python_ext, - paths=env_etc.prime_common_includes) - Export("env_prime_boost_python_ext") - -SConscript("index_ambiguity/SConscript") diff --git a/prime/__init__.py b/prime/__init__.py deleted file mode 100644 index 564cbadf27c..00000000000 --- a/prime/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import absolute_import, division, print_function -import boost_adaptbx.boost.python as bp -ext = bp.import_ext("prime_ext") -from prime_ext import * - -prime_description = ''' The Post-RefInement and MErging (PRIME) program for -the scaling, merging and post-refinement of integrated diffraction images. - -Reference: Uervirojnangkoorn, et al., eLife, 2015''' - -prime_license = ''' PRIME is distributed under open source license''' diff --git a/prime/command_line/__init__.py b/prime/command_line/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/prime/command_line/change_basis.py b/prime/command_line/change_basis.py deleted file mode 100644 index 693fcfc1dc9..00000000000 --- a/prime/command_line/change_basis.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import absolute_import, division, print_function -# LIBTBX_SET_DISPATCHER_NAME prime.change_basis - -import argparse -from iotbx import reflection_file_reader -from cctbx import sgtbx -import os - -def main(mtz, index_basis): - # read in mtz - reflection_file = reflection_file_reader.any_reflection_file(mtz) - miller_arrays = reflection_file.as_miller_arrays() - # apply new basis - cb_op = sgtbx.change_of_basis_op(index_basis) - miller_array_new = miller_arrays[0].change_basis(cb_op) - # write out new mtz - hklout = os.path.splitext(mtz)[0]+'_modified.mtz' - mtz_dataset = miller_array_new.as_mtz_dataset(column_root_label="IOBS") - mtz_dataset.mtz_object().write(file_name=hklout) - print('Output file saved to', hklout) - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Convert an mtz file to the specified basis.' - ) - parser.add_argument( - 'mtz', - metavar='MTZ File', - help='Path to an mtz file' - ) - parser.add_argument( - 'index_basis', - metavar='New Basis', - help='New index basis operators' - ) - args = parser.parse_args() - print('Convert ', args.mtz, ' to ', args.index_basis.strip()) - main(args.mtz, args.index_basis.strip()) diff --git a/prime/command_line/change_path_indexing_ambiguity.py b/prime/command_line/change_path_indexing_ambiguity.py deleted file mode 100644 index 3ecc19793fa..00000000000 --- a/prime/command_line/change_path_indexing_ambiguity.py +++ /dev/null @@ -1,62 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.change_path_indexing_ambiguity -""" -Author : Uervirojnangkoorn, M. -Created : 8/20/2015 -Description : read indexing_ambiguity pickle and overwrite oldpath with new path -""" -from __future__ import absolute_import, division, print_function -import sys -from six.moves import cPickle as pickle -from six.moves import range - -def read_input(args): - data = '' - oldpath = '' - newpath = '' - for i in range(len(args)): - if args[i]=='-h': - print(txt_help) - exit() - - pair=args[i].split('=') - if len(pair) == 2: - if pair[0]=='data': - data = pair[1] - elif pair[0]=='oldpath': - oldpath = pair[1] - elif pair[0]=='newpath': - newpath = pair[1] - - if data == '' or oldpath == '' or newpath == '': - print("Please all parameters data, oldpath, and newpath.") - exit() - - return data, oldpath, newpath - - -if (__name__ == "__main__"): - #Help message - txt_help = "Use this command to change cctbx.xfel integration pickle path to a new path\n" - txt_help += "in case they have been moved to a new location.\n" - txt_help += "Usage: prime.change_path_indexing_ambiguity data=indexing_ambiguity.pickle oldpath=/old/path/to/pickles newpath=/new/path/to/pickles\n" - txt_help += "Good luck!\n" - - #Read input parameters and frames (pickle files) - if len(sys.argv) == 1: - print(txt_help) - exit() - - data, oldpath, newpath = read_input(args = sys.argv[1:]) - - pickle_data = pickle.load(open(data, "rb")) - new_pickle_data = {} - cn_i = 0 - for key in pickle_data.keys(): - basis = pickle_data[key] - newkey = key.replace(oldpath, newpath) - new_pickle_data[newkey] = basis - print(cn_i+1, newkey, basis) - cn_i +=1 - - pickle.dump(new_pickle_data, open("new_indexing_ambiguity.pickle", "wb")) - print('Found and replace %6d keys'%(cn_i)) diff --git a/prime/command_line/cluster_isoform.py b/prime/command_line/cluster_isoform.py deleted file mode 100644 index 1c75b0d8a91..00000000000 --- a/prime/command_line/cluster_isoform.py +++ /dev/null @@ -1,213 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.cluster_isoform -""" cluster diffraction images by """ -from __future__ import absolute_import, division, print_function -import six -from six.moves import range -from six.moves import zip -__author__ = 'Monarin Uervirojnangkoorn, monarin@gmail.com' - -from prime.isoform_cluster.mod_isoform_cluster import isoform_cluster_handler -from prime.index_ambiguity.mod_kmeans import kmeans_handler -from prime.postrefine.mod_mx import mx_handler -from six.moves import cPickle as pickle -from libtbx.easy_mp import pool_map -import numpy as np -import random, os, sys - -def solve_with_mtz_mproc(args): - frame_no, pickle_filename, main_obs, iparams, miller_array_ref_set = args - isoch = isoform_cluster_handler() - cluster_id, txt_out = isoch.calc_cc(frame_no, pickle_filename, main_obs, iparams, miller_array_ref_set) - print(txt_out) - return pickle_filename, cluster_id - -def calculate_r_mproc(args): - frame_no, pickle_filename, obs_main, obs_list = args - isoch = isoform_cluster_handler() - r_set, main_obs, txt_out = isoch.calc_r(frame_no, pickle_filename, obs_main, obs_list) - print(txt_out) - return pickle_filename, r_set, main_obs - -def get_obs_mproc(args): - frame_no, pickle_filename, iparams = args - isoch = isoform_cluster_handler() - obs_asu = isoch.get_observations(pickle_filename, iparams) - return obs_asu, pickle_filename - -def write_out_solutions(iparams, sol_pickle): - cluster_files = [os.path.join(iparams.run_no,'cluster_'+str(i)+'.lst') for i in range(iparams.isoform_cluster.n_clusters)] - cluster_text_set = ['']*iparams.isoform_cluster.n_clusters - for frame, cluster_id in six.iteritems(sol_pickle): - cluster_text_set[cluster_id] += frame + '\n' - for i in range(iparams.isoform_cluster.n_clusters): - with open(cluster_files[i], 'w') as f: f.write(cluster_text_set[i]) - -class isoform_cluster_run_handler(object): - def __init__(self): - self.module_name = "isoform_cluster" - - def get_observation_set(self, iparams, frame_files, n_sel_frames): - n_frames = len(frame_files) - frames = [(i, frame_files[i], iparams) for i in random.sample(range(n_frames), n_sel_frames)] - #get observations list - print("Reading observations") - obs_results = pool_map( - iterable=frames, - func=get_obs_mproc, - processes=iparams.n_processors) - frame_files_sel = [] - obs_list = [] - for result in obs_results: - if result: - main_asu, pickle_filename = result - if main_asu: - frame_files_sel.append(pickle_filename) - obs_list.append(main_asu) - return frame_files_sel, obs_list - - def run(self, args): - #read inputs - from prime.postrefine.mod_input import process_input, read_pickles - iparams, txt_out_input = process_input(args) - print(txt_out_input) - with open(os.path.join(iparams.run_no,self.module_name,'log.txt'), 'w') as f: f.write(txt_out_input) - #read all integration pickles - frame_files = read_pickles(iparams.data) - n_frames = len(frame_files) - if n_frames == 0: - print("No integration pickle found. Exit program.") - return None, iparams - #start - if iparams.isoform_cluster.isorefin: - #get collection of iso. ref. reflection set. - mxh = mx_handler() - miller_array_ref_set = [] - for isorefin in iparams.isoform_cluster.isorefin: - flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(isorefin) - if flag_ref_found: miller_array_ref_set.append(miller_array_ref) - #get observation list - frame_files_sel, obs_list = self.get_observation_set(iparams, frame_files, n_frames) - if miller_array_ref_set: - frames = [(i, frame_files_sel[i], obs_list[i], iparams, miller_array_ref_set) for i in range(len(obs_list))] - cc_results = pool_map( - iterable=frames, - func=solve_with_mtz_mproc, - processes=iparams.n_processors) - sol_pickle = {} - for result in cc_results: - pickle_filename, cluster_id = result - sol_pickle[pickle_filename] = cluster_id - write_out_solutions(iparams, sol_pickle) - txt_out = "Cluster images with given "+str(len(miller_array_ref_set))+" mtz files completed. Use cluster_0.lst - cluster_k.lst (for k clusters) for merging.\n" - print(txt_out) - with open(os.path.join(iparams.run_no,self.module_name,'log.txt'), 'a') as f: f.write(txt_out) - return - - #************************************************* - #solve with Brehm & Diederichs - sample size n_sample_frames then bootstrap the rest - txt_out = "Cluster images with B&D algorithms.\n" - frame_files_sel, obs_list = self.get_observation_set(iparams, frame_files, iparams.isoform_cluster.n_sample_frames) - frames = [(i, frame_files_sel[i], obs_list[i], obs_list) for i in range(len(frame_files_sel))] - #calculate r - print("Calculating R") - calc_r_results = pool_map( - iterable=frames, - func=calculate_r_mproc, - processes=iparams.n_processors) - frame_files_sel = [] - r_matrix = [] - obs_list = [] - for result in calc_r_results: - if result: - pickle_filename, r_set, obs = result - frame_files_sel.append(pickle_filename) - obs_list.append(obs) - if len(r_matrix) == 0: - r_matrix = r_set - else: - r_matrix = np.append(r_matrix, r_set, axis=0) - #choose groups with best R - print("Selecting frames with best R") - i_mean_r = np.argsort(np.mean(r_matrix, axis=1))[::-1] - r_matrix_sorted = r_matrix[i_mean_r] - frame_files_sorted = np.array(frame_files_sel)[i_mean_r] - obs_list_sorted = np.array(obs_list)[i_mean_r] - frame_files_sel = [] - obs_sel = [] - for frame_file, r_set, obs in zip(frame_files_sorted, r_matrix_sorted, obs_list_sorted): - if frame_file not in frame_files_sel: - frame_files_sel.append(frame_file) - obs_sel.append(obs) - print(frame_file, np.mean(r_set)) - if len(frame_files_sel) >= iparams.isoform_cluster.n_selected_frames: - print('Found all %6.0f good frames'%(len(frame_files_sel))) - break - #Recalculate r for the new selected list - frames = [(i, frame_files_sel[i], obs_sel[i], obs_sel) for i in range(len(frame_files_sel))] - print("Re-calculating R") - calc_r_results = pool_map( - iterable=frames, - func=calculate_r_mproc, - processes=iparams.n_processors) - frame_files_sel = [] - r_matrix = [] - obs_list = [] - for result in calc_r_results: - if result: - pickle_filename, r_set, obs = result - frame_files_sel.append(pickle_filename) - obs_list.append(obs) - if len(r_matrix) == 0: - r_matrix = r_set - else: - r_matrix = np.append(r_matrix, r_set, axis=0) - print("Minimizing frame distance") - isoch = isoform_cluster_handler() - x_set = isoch.optimize(r_matrix, flag_plot=iparams.flag_plot) - print("Clustering results") - kmh = kmeans_handler() - k = iparams.isoform_cluster.n_clusters - centroids, labels = kmh.run(x_set, k, flag_plot=iparams.flag_plot) - print("Get solution pickle and cluster files list") - sol_pickle, cluster_files = isoch.assign_cluster(frame_files_sel, labels, k, \ - os.path.join(iparams.run_no,self.module_name)) - #if more frames found, merge the sample frames to get a reference set - #that can be used for breaking the ambiguity. - if n_frames > iparams.isoform_cluster.n_selected_frames: - print("Assign cluster_id for the remaining images.") - old_iparams_data = iparams.data[:] - miller_array_ref_set = [] - from prime.command_line.postrefine import scale_frames, merge_frames - for i in range(k): - #generate a reference set from solved frames - with open(cluster_files[i]) as f: frame_files_processed = f.read().split('\n')[:-1] - scaled_pres_set = scale_frames(range(len(frame_files_processed)), frame_files_processed, iparams) - mdh, txt_merge_out = merge_frames(scaled_pres_set, iparams, \ - mtz_out_prefix=os.path.join(self.module_name,'cluster_'+str(i))) - miller_array_ref_set.append(mdh.miller_array_merge) - txt_out += txt_merge_out - #setup a list of remaining frames - frame_files_remain = [frame for frame in frame_files if frame not in sol_pickle] - frame_files_remain_sel, obs_remain_sel_list = self.get_observation_set(iparams, \ - frame_files_remain, len(frame_files_remain)) - frames = [(i, frame_files_remain_sel[i], obs_remain_sel_list[i], iparams, miller_array_ref_set) for i in range(len(obs_remain_sel_list))] - cc_results = pool_map( - iterable=frames, - func=solve_with_mtz_mproc, - processes=iparams.n_processors) - for result in cc_results: - if result: - pickle_filename, cluster_id = result - sol_pickle[pickle_filename] = cluster_id - iparams.data = old_iparams_data[:] - #write out solution pickle - write_out_solutions(iparams, sol_pickle) - #write out text output - txt = "Cluster images completed. Use cluster_0.lst - cluster_k.lst (for k clusters) for merging.\n" - txt_out += txt - print(txt) - with open(os.path.join(iparams.run_no,self.module_name,'log.txt'), 'a') as f: f.write(txt_out) - -if (__name__ == "__main__"): - isoc_runh = isoform_cluster_run_handler() - isoc_runh.run(sys.argv[1:] if len(sys.argv) > 1 else None) diff --git a/prime/command_line/compare_solution_pickles.py b/prime/command_line/compare_solution_pickles.py deleted file mode 100644 index 43d923db399..00000000000 --- a/prime/command_line/compare_solution_pickles.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import absolute_import, division, print_function -# LIBTBX_SET_DISPATCHER_NAME prime.compare_solution_pickles - -from six.moves import cPickle as pickle -import argparse - -def main(sol_fname, ind_fname): - sol_pickle = pickle.load(open(sol_fname, "rb")) - ind_pickle = pickle.load(open(ind_fname, "rb")) - cn_match = 0 - for key in sol_pickle: - if key in ind_pickle: - if sol_pickle[key] == ind_pickle[key]: - cn_match += 1 - else: - print(key, sol_pickle[key], ind_pickle[key]) - print('Found %d images with %d matches'%(len(sol_pickle), cn_match)) - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Compare two indexing ambiguity solution pickles' - ) - parser.add_argument( - 'pickle1', - metavar='First Pickle', - help='Path to the first solution pickle' - ) - parser.add_argument( - 'pickle2', - metavar='Second pickle', - help='Path to the second solution pickle' - ) - args = parser.parse_args() - print("Compare two solution pickles.") - main(args.pickle1, args.pickle2) diff --git a/prime/command_line/comparestats.py b/prime/command_line/comparestats.py deleted file mode 100644 index 71d2ba727e2..00000000000 --- a/prime/command_line/comparestats.py +++ /dev/null @@ -1,95 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.comparestats -''' -Author : Uervirojnangkoorn, M. -Created : 6/25/2016 -Description : Given list of log files, compare completeness, n_obs, rmerge, and cc12. -''' -from __future__ import absolute_import, division, print_function -import matplotlib.pyplot as plt -import sys - -if len(sys.argv)==1: - print('Use prime.comparestats to compare stats from different log files.') - print('Usage: prime.comparestats log1.txt log2.txt log3.txt c=cycle_no n=n_bins') - exit() -log_files = [] -cycle_no = 0 -for arg in sys.argv[1:]: - param = arg.split('=') - if len(param) == 1: - log_files.append(param[0]) - elif len(param) == 2: - if param[0]=='c': - cycle_no = int(param[1]) - elif param[0] =='n': - n_bins = int(param[1]) - -data_dict = {'onedsqr':[], 'complete':[], 'n_obs':[], 'r_merge':[], 'cc12':[]} -for log_file in log_files: - pf = open(log_file,'r') - data = pf.read().split('\n') - if cycle_no == 0: - search_word = '/mean_scaled_merge.mtz' - else: - search_word = '/postref_cycle_'+str(cycle_no)+'_merge.mtz' - cn_row = 0 - for data_row in data: - if data_row.endswith(search_word): - found_row = cn_row + 3 - break - cn_row += 1 - onedsqr = [] - complete = [] - n_obs = [] - r_merge = [] - cc12 = [] - for data_row in data[found_row:found_row+n_bins]: - data_col = data_row.split() - onedsqr.append(1/(float(data_col[3])**2)) - complete.append(float(data_col[4])) - n_obs.append(float(data_col[8])) - r_merge.append(float(data_col[9])) - cc12.append(float(data_col[11])) - data_dict['onedsqr'].append(onedsqr) - data_dict['complete'].append(complete) - data_dict['n_obs'].append(n_obs) - data_dict['r_merge'].append(r_merge) - data_dict['cc12'].append(cc12) - -plt.subplot(221) -x = data_dict['onedsqr'][0] -for y in data_dict['complete']: - plt.plot(x,y) -plt.legend(log_files, loc=[0,0]) -plt.title('Completeness') -#plt.xlabel('1/d^2') -plt.grid(True) -#plt.ylim([95,100]) -plt.subplot(222) -x = data_dict['onedsqr'][0] -for y in data_dict['n_obs']: - plt.plot(x,y) -#plt.legend(log_files, loc=[0,0]) -plt.title('No. observations') -#plt.xlabel('1/d^2') -plt.grid(True) -#plt.ylim([95,100]) -plt.subplot(223) -x = data_dict['onedsqr'][0] -for y in data_dict['r_merge']: - plt.plot(x,y) -#plt.legend(log_files, loc=[0,0]) -plt.title('Rmerge') -plt.xlabel('1/d^2') -plt.grid(True) -#plt.ylim([95,100]) -plt.subplot(224) -x = data_dict['onedsqr'][0] -for y in data_dict['cc12']: - plt.plot(x,y) -#plt.legend(log_files, loc=[0,0]) -plt.title('CC1/2') -plt.xlabel('1/d^2') -plt.grid(True) -#plt.ylim([95,100]) -plt.show() diff --git a/prime/command_line/explore_twin_operators.py b/prime/command_line/explore_twin_operators.py deleted file mode 100644 index 9bb83e0dfe3..00000000000 --- a/prime/command_line/explore_twin_operators.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import absolute_import, division, print_function -# LIBTBX_SET_DISPATCHER_NAME prime.explore_twin_operators - -from prime.index_ambiguity.mod_indexing_ambiguity import indamb_handler -import argparse, os -from prime.postrefine.mod_input import read_frame, read_pickles -import six - -def main(data, only_merohedral): - indambh = indamb_handler() - intFileList = read_pickles([data]) - if intFileList: - obsList = {} - for intFileName in intFileList: - intPickle = read_frame(intFileName) - try: - obs = intPickle['observations'][0] - obsList[intFileName] = obs - except Exception as e: - print("Warning:", e) - pass - for key,value in six.iteritems(obsList): - if only_merohedral: - flag_all = False - else: - flag_all = True - ops = indambh.generate_twin_operators(value, flag_all=flag_all) - if ops: - print(os.path.basename(key), '%6.1f,%6.1f,%6.1f,%6.1f,%6.1f,%6.1f'%value.unit_cell().parameters(), ' '.join([op.operator.r().as_hkl() for op in ops])) - else: - print(os.path.basename(key), '%6.1f,%6.1f,%6.1f,%6.1f,%6.1f,%6.1f'%value.unit_cell().parameters(), 'Twining operators not found') - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Explore (pseudo)merohedral twinning operators' - ) - parser.add_argument( - 'integration_pickles', - metavar='Integration Pickles', - help='Path to integration pickles (regex. is acceptable).' - ) - parser.add_argument('--only_merohedral', dest='only_merohedral', action='store_const', - const=True, default=False, - help='Flag for showing only merohedral twinnings') - args = parser.parse_args() - if args.only_merohedral: - print("Only showing results with merohedral twinning operators") - else: - print("Showing all possible pseudo- and true-merohedral twinning operators") - args = parser.parse_args() - - main(args.integration_pickles, args.only_merohedral) diff --git a/prime/command_line/frame_extractor_by_scan.py b/prime/command_line/frame_extractor_by_scan.py deleted file mode 100644 index ef1bf35bff3..00000000000 --- a/prime/command_line/frame_extractor_by_scan.py +++ /dev/null @@ -1,225 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.frame_extractor -""" -Author : Uervirojnangkoorn, M. -Desc : Taking the original code from xfel/command_line/frame_extractor.py - and adding by scan so that each scan is output as a single pickle file. -""" -from __future__ import absolute_import, division, print_function - -from dials.array_family import flex -from dials.util.options import Importer, flatten_reflections, flatten_experiments, ArgumentParser -from cctbx import crystal, miller -from cctbx.crystal_orientation import crystal_orientation -import iotbx.phil -import cctbx, os -from libtbx import easy_pickle -from six.moves import range - -class ConstructFrame(object): - def get_template_pickle(self): - return {'current_cb_op_to_primitive': 0, - 'current_orientation':0, - 'distance':0, - 'effective_tiling':0, - 'mapped_predictions':[[]], - 'max_signal':0, - 'ML_domain_size_ang':[0], - 'ML_half_mosaicity_deg':[0], - 'mosaicity':0, - 'model_partialities':[None], - 'observations':[0], - 'pointgroup':0, - 'residual':0, - 'sa_parameters':['None'], - 'wavelength':0, - 'xbeam':0, - 'ybeam':0} - - def __init__(self, reflections, experiment, scan_no): - # assemble template and unpack files - self.frame = self.get_template_pickle() - self.pixel_size = experiment.detector[0].get_pixel_size()[0] - - if 'intensity.prf.value' in reflections: - self.method = 'prf' # integration by profile fitting - elif 'intensity.sum.value' in reflections: - self.method = 'sum' # integration by simple summation - #self.reflections = reflections.select(reflections['intensity.' + self.method + '.variance'] > 0) # keep only spots with sigmas above zero - self.reflections = reflections - self.scan_no = scan_no - self.reflections = self.reflections.select((self.reflections['xyzobs.px.value'].parts()[2] >= scan_no) & (self.reflections['xyzobs.px.value'].parts()[2] < scan_no+1)) #select only reflections in the scan no. - self.xtal = experiment.crystal - self.beam_obj = experiment.beam - self.det = experiment.detector - self.gonio = experiment.goniometer - self.scan = experiment.scan - self.img_sweep = experiment.imageset - print(scan_no, len(self.reflections)) - - # experiment-dependent components --------------------------------------------------------------------------- - - # get wavelength - def populate_wavelength(self): - assert self.beam_obj.get_wavelength() is not None, "no wavelength" - self.frame['wavelength'] = self.beam_obj.get_wavelength() - - # get detector distance in mm - def populate_distance(self): - assert self.det[0].get_distance() is not None, "no detector distance" - self.frame['distance'] = self.det[0].get_distance() - - # get xbeam and ybeam in mm - def populate_beam_dir(self): - assert self.beam_obj.get_s0() is not None, "no beam direction" - self.frame['xbeam'], self.frame['ybeam'] = self.det[0].get_beam_centre(self.beam_obj.get_s0()) - - # get max signal - def populate_max_signal(self): - pass - - # get effective tiling - def populate_effective_tiling(self): - pass - - # indicate simulated annealing parameters, if present - def populate_sa_params(self): - pass - - # crystal-dependent components ------------------------------------------------------------------------------ - - # generate a crystal orientation object from the A* matrix - def populate_orientation(self): - assert self.xtal.get_A_at_scan_point(self.scan_no) is not None, "no crystal orientation matrix" - self.frame['current_orientation'] = [crystal_orientation(self.xtal.get_A_at_scan_point(self.scan_no).elems, True)] - - # generate change-of-basis operation for current to primitive cell - def populate_op_to_primitive(self): - assert self.xtal.get_space_group() is not None, "no space group" - self.frame['current_cb_op_to_primitive'] = [self.xtal.get_space_group().z2p_op()] - - # fetch the point group associated with the crystal - def populate_point_group(self): - assert self.xtal.get_space_group() is not None, "no space group" - self.frame['pointgroup'] = str(self.xtal.get_space_group().build_derived_point_group().info()) - - # get mosaicity - def populate_mosaicity(self): - try: - self.frame['mosaicity'] = self.xtal.get_mosaicity() - except AttributeError as e: - pass - - # get any available ML values - def populate_ML_values(self): - try: - self.frame['ML_half_mosaicity_deg'] = [self.xtal.get_half_mosaicity_deg()] - except AttributeError: - pass - try: - self.frame['ML_domain_size_ang'] = [self.xtal.get_domain_size_ang()] - except AttributeError: - pass - - # observations-dependent components ------------------------------------------------------------------------- - - # generate a miller array containing the Miller indices, intensities and variances for one frame - def populate_observations(self): - intensities = self.reflections['intensity.' + self.method + '.value'] - variances = self.reflections['intensity.' + self.method + '.variance'] - space_group = crystal.symmetry(self.xtal.get_unit_cell(), str(self.xtal.get_space_group().info())) - miller_set = miller.set(space_group, self.reflections['miller_index']) - self.frame['observations'][0] = cctbx.miller.array(miller_set, intensities, flex.sqrt(variances)).set_observation_type_xray_intensity() - - # collect predicted spot positions - def populate_pixel_positions(self): - assert 'xyzcal.px' in self.reflections, "no calculated spot positions" - self.frame['mapped_predictions'][0] = flex.vec2_double() - for i in range(len(self.reflections['xyzcal.px'])): - self.frame['mapped_predictions'][0].append(tuple(self.reflections['xyzcal.px'][i][1::-1])) # 1::-1 reverses the order taking only the first two elements first. - - # generate a list of dictionaries containing a series of corrections for each predicted reflection - def populate_corrections(self): - assert 'xyzobs.px.value' in self.reflections and 'xyzcal.px' in self.reflections, "no calculated or observed spot positions" - assert self.frame['xbeam'] != 0 and self.frame['ybeam'] != 0, "invalid beam center" - self.frame['correction_vectors'] = [[]] - for idx in range(len(self.reflections['xyzobs.px.value'])): - if self.reflections['xyzcal.px'][idx][0:2] != self.reflections['xyzobs.px.value'][idx][0:2]: - theoret_center = 1765/2, 1765/2 - refined_center = self.frame['xbeam']/self.pixel_size, self.frame['ybeam']/self.pixel_size # px to mm conversion - hkl = self.reflections['miller_index'][idx] - obsspot = tuple(self.reflections['xyzobs.px.value'][idx][0:2]) - predspot = tuple(self.reflections['xyzcal.px'][idx][0:2]) - self.frame['correction_vectors'][0].append({'refinedcenter':refined_center, 'hkl':hkl, 'setting_id':0, 'azimuthal':0, 'radial':0, - 'obsspot':obsspot, 'obscenter':theoret_center, 'predspot':predspot}) - - # get partialities - def populate_partialities(self): - pass - - # produce residuals - def populate_residuals(self): - pass - - # combine all of the above - def make_frame(self): - self.populate_wavelength() - self.populate_distance() - self.populate_beam_dir() - self.populate_max_signal() - self.populate_effective_tiling() - self.populate_sa_params() - self.populate_orientation() - self.populate_op_to_primitive() - self.populate_point_group() - self.populate_mosaicity() - self.populate_ML_values() - self.populate_observations() - self.populate_pixel_positions() - # self.populate_corrections() # works, but unnecessary - self.populate_partialities() - self.populate_residuals() - return self.frame - -class ConstructFrameFromFiles(ConstructFrame): - def __init__(self, pickle_name, json_name, scan_no): - # load the integration.pickle file (reflection table) into memory and - # load the experiments.json file (json) into memory, piecewise. - # check_format=False because we don't wont to load any imagesets in the - # experiement list - importer = Importer([pickle_name, json_name], read_experiments=True, read_reflections=True, check_format=False) - if importer.unhandled: - print("unable to process:", importer.unhandled) - ConstructFrame.__init__(self, flatten_reflections(importer.reflections)[0], - flatten_experiments(importer.experiments)[0], - scan_no) - -if __name__ == "__main__": - master_phil_scope = iotbx.phil.parse(""" - pickle_name = None - .type = path - .help = path to a reflection table (integrated.pickle) file - json_name = None - .type = path - .help = path to an experiments.json file - output_dir = None - .type = path - .help = if set, path to directory to save the new pickle file - """) - parser = ArgumentParser(phil=master_phil_scope) - params, options = parser.parse_args(show_diff_phil=True) - #get scan range number - importer = Importer([params.pickle_name, params.json_name], read_experiments=True, read_reflections=True, check_format=False) - if importer.unhandled: - print("unable to process:", importer.unhandled) - experiment = flatten_experiments(importer.experiments)[0] - scan = experiment.scan - for scan_no in range(scan.get_image_range()[0], scan.get_image_range()[1]): - #build each frame - frame = ConstructFrameFromFiles(params.pickle_name, params.json_name, scan_no).make_frame() - if not params.output_dir is None: - assert os.path.isdir(params.output_dir) - basename = os.path.basename(params.pickle_name) - name = os.path.splitext(basename)[0] + "_extracted_"+str(scan_no)+".pickle" - dest_path = os.path.join(params.output_dir, name) - assert not os.path.isfile(dest_path) - easy_pickle.dump(dest_path, frame) diff --git a/prime/command_line/iotacc.py b/prime/command_line/iotacc.py deleted file mode 100644 index ffa5dba56c6..00000000000 --- a/prime/command_line/iotacc.py +++ /dev/null @@ -1,335 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.iotacc -''' -Author : Uervirojnangkoorn, M. -Created : 11/25/2014 -Description : iotacc selects iota integration results base on CC with ref. set. -''' -from __future__ import absolute_import, division, print_function -import os, sys -import numpy as np -import math -from iotbx import reflection_file_reader -from cctbx.array_family import flex -from cctbx import miller -from six.moves import cPickle as pickle -from cctbx.crystal import symmetry -from scitbx.matrix import sqr -import shutil -from libtbx.easy_mp import pool_map -from prime.postrefine.mod_input import read_frame -from six.moves import range - -def read_input(args): - from prime.postrefine.mod_input import process_input - iparams, txt_out_input = process_input(args) - return iparams, txt_out_input - -def read_pickles(data): - frame_files = [] - for file_name in os.listdir(data): - if file_name.endswith('.pickle'): - frame_files.append(data+'/'+file_name) - - return frame_files - -def get_Eoc_corrected_observations(pickle_filename, iparams): - ''' - Read pickle file name and output Eoc corrected original miller array object. - ''' - observations_pickle = read_frame(pickle_filename) - observations_original = observations_pickle["observations"][0] - wavelength = observations_pickle["wavelength"] - crystal_init_orientation = observations_pickle["current_orientation"][0] - uc_params = observations_original.unit_cell().parameters() - detector_distance_mm = observations_pickle['distance'] - mm_predictions = iparams.pixel_size_mm*(observations_pickle['mapped_predictions'][0]) - xbeam = observations_pickle["xbeam"] - ybeam = observations_pickle["ybeam"] - alpha_angle = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ - for pred in mm_predictions]) - spot_pred_x_mm = flex.double([pred[0]-xbeam for pred in mm_predictions]) - spot_pred_y_mm = flex.double([pred[1]-ybeam for pred in mm_predictions]) - - #filter resolution - i_sel_res = observations_original.resolution_filter_selection(\ - d_max=iparams.iotacc.d_max, d_min=iparams.iotacc.d_min) - observations_original = observations_original.customized_copy(\ - indices=observations_original.indices().select(i_sel_res), \ - data=observations_original.data().select(i_sel_res), \ - sigmas=observations_original.sigmas().select(i_sel_res)) - alpha_angle = alpha_angle.select(i_sel_res) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res) - - - #Filter weak - i_sel = (observations_original.data()/observations_original.sigmas()) > iparams.iotacc.sigma_min - observations_original = observations_original.customized_copy(\ - indices=observations_original.indices().select(i_sel),\ - data=observations_original.data().select(i_sel),\ - sigmas=observations_original.sigmas().select(i_sel)) - alpha_angle = alpha_angle.select(i_sel) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel) - - #calculate spot radius - from prime.postrefine.mod_leastsqr import calc_spot_radius - spot_radius = calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), - observations_original.indices(), wavelength) - - G = 1 - B = 0 - from prime.postrefine.mod_leastsqr import calc_partiality_anisotropy_set - two_theta = observations_original.two_theta(wavelength=wavelength).data() - sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() - ry = 0 - rz = 0 - re = iparams.gamma_e - rotx = 0 - roty = 0 - - #calc partiality - partiality_init, delta_xy_init, rs_init, rh_init = calc_partiality_anisotropy_set(crystal_init_orientation.unit_cell(), rotx, roty, observations_original.indices(), ry, rz, spot_radius, re, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) - - from prime.postrefine.mod_leastsqr import calc_full_refl - I_full = calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, - G, B, partiality_init, rs_init, iparams.flag_volume_correction) - sigI_full = calc_full_refl(observations_original.sigmas(), sin_theta_over_lambda_sq, - G, B, partiality_init, rs_init, iparams.flag_volume_correction) - observations_Eoc_corrected = observations_original.customized_copy(data=I_full, sigmas=sigI_full) - - return observations_Eoc_corrected, observations_original - -def get_observations_ref(hklrefin): - ''' - Read and return ref. observations. - ''' - reflection_file_iso = reflection_file_reader.any_reflection_file(hklrefin) - miller_arrays_iso=reflection_file_iso.as_miller_arrays() - is_found_iso_as_intensity_array = False - is_found_iso_as_amplitude_array = False - miller_array_iso = None - for miller_array in miller_arrays_iso: - if miller_array.is_xray_intensity_array(): - miller_array_iso = miller_array.deep_copy() - is_found_iso_as_intensity_array = True - break - elif miller_array.is_xray_amplitude_array(): - is_found_iso_as_amplitude_array = True - miller_array_converted_to_intensity = miller_array.as_intensity_array() - if is_found_iso_as_intensity_array == False: - if is_found_iso_as_amplitude_array: - miller_array_iso = miller_array_converted_to_intensity.deep_copy() - else: - flag_hklisoin_found = False - - return miller_array_iso - -def calc_cc(miller_array_ref, miller_array_obs): - ''' - Calculate cc between matched intensities. - ''' - matches_ref = miller.match_multi_indices( - miller_indices_unique=miller_array_ref.indices(), - miller_indices=miller_array_obs.indices()) - - I_ref = flex.double([miller_array_ref.data()[pair[0]] for pair in matches_ref.pairs()]) - I_o = flex.double([miller_array_obs.data()[pair[1]] for pair in matches_ref.pairs()]) - - cc = 0 - n_refl = 0 - if len(matches_ref.pairs()) > 0 : - cc = np.corrcoef(I_o, I_ref)[0,1] - if math.isnan(cc): - cc = 0 - n_refl = len(matches_ref.pairs()) - - return cc, n_refl - -def good_unit_cell(uc_params, target_unit_cell, uc_tol): - ''' - check unit cell - ''' - flag_good_uc = False - if (abs(uc_params[0]-target_unit_cell[0]) \ - <= (uc_tol*target_unit_cell[0]/100) \ - and abs(uc_params[1]-target_unit_cell[1]) \ - <= (uc_tol*target_unit_cell[1]/100) \ - and abs(uc_params[2]-target_unit_cell[2]) \ - <= (uc_tol*target_unit_cell[2]/100) \ - and abs(uc_params[3]-target_unit_cell[3]) \ - <= (uc_tol*target_unit_cell[3]/100) \ - and abs(uc_params[4]-target_unit_cell[4]) \ - <= (uc_tol*target_unit_cell[4]/100) \ - and abs(uc_params[5]-target_unit_cell[5]) \ - <= (uc_tol*target_unit_cell[5]/100)): - flag_good_uc = True - return flag_good_uc - -def select_best_by_cc_mproc(shot_no, cryst_id, iparams): - if iparams.iotacc.set_id is not None: - data_dir = iparams.data[0] + '/' +iparams.iotacc.set_id - else: - data_dir = iparams.data[0] - - cryst_id_shot_no = cryst_id+'_0_'+str(shot_no).zfill(iparams.iotacc.LEN_SHOT_SEQ) - #cryst_id_shot_no = cryst_id+str(shot_no).zfill(LEN_SHOT_SEQ) - cc_list = flex.double() - n_refl_list = flex.int() - pickle_filename_list = [] - flag_success = False - d_min, d_max = (iparams.iotacc.d_min, iparams.iotacc.d_max) - txt_out = 'ID'.center(3)+'shot'.center(6)+'res.'.center(7)+'n_refl'.center(7)+ \ - 'cc_pt'.center(7)+'cc_fl'.center(7)+'n_ref'.center(7)+'cc_pt'.center(7)+'cc_fl'.center(7)+'n_iso'.center(7) + \ - 'a'.center(7)+'b'.center(7)+'c'.center(7)+'alp'.center(7)+'gam'.center(7)+'bet'.center(7)+'file name'.center(20)+'\n' - for tmp_shot_dir in os.listdir(data_dir+'/'+cryst_id): - if tmp_shot_dir.find(cryst_id_shot_no) > 0: - data = data_dir+'/'+cryst_id+'/'+tmp_shot_dir - #data = data_dir+'/'+tmp_shot_dir - frame_files = [] - if os.path.isdir(data): - frame_files = read_pickles(data) - if len(frame_files)==0: - print(data, ' - no pickle file found.') - else: - for pickle_filename in frame_files: - pickle_filename_split = pickle_filename.split('/') - pickle_filename_only = pickle_filename_split[len(pickle_filename_split)-1] - pickle_filename_only_split = pickle_filename_only.split('_') - - observations_Eoc_corrected, observations_original = get_Eoc_corrected_observations(pickle_filename, iparams) - - #check unitcell - uc_params = observations_Eoc_corrected.unit_cell().parameters() - cc_eoc, n_refl_eoc, cc_ori, n_refl_ori, cc_iso_eoc, n_refl_iso_eoc, cc_iso_ori, n_refl_iso_ori = (0,0,0,0,0,0,0,0) - a,b,c,alpha,beta,gamma = uc_params - if good_unit_cell(uc_params, iparams.target_unit_cell.parameters(), iparams.iotacc.uc_tolerance): - try: - miller_set = symmetry( - unit_cell=observations_Eoc_corrected.unit_cell().parameters(), - space_group_symbol=str(iparams.target_space_group) - ).build_miller_set( - anomalous_flag=iparams.target_anomalous_flag, - d_min=d_min) - - observations_Eoc_corrected = observations_Eoc_corrected.customized_copy(anomalous_flag=iparams.target_anomalous_flag, - crystal_symmetry=miller_set.crystal_symmetry()) - - miller_set = symmetry( - unit_cell=observations_original.unit_cell().parameters(), - space_group_symbol=str(iparams.target_space_group) - ).build_miller_set( - anomalous_flag=iparams.target_anomalous_flag, - d_min=d_min) - - observations_original = observations_original.customized_copy(anomalous_flag=iparams.target_anomalous_flag, - crystal_symmetry=miller_set.crystal_symmetry()) - - #convert to asymmetric unit index - observations_Eoc_corrected_asu = observations_Eoc_corrected.map_to_asu() - observations_original_asu = observations_original.map_to_asu() - - #calculate CCori CCEoc - miller_array_ref = get_observations_ref(iparams.hklrefin) - cc_eoc, n_refl_eoc = calc_cc(miller_array_ref, observations_Eoc_corrected_asu) - cc_ori, n_refl_ori = calc_cc(miller_array_ref, observations_original_asu) - - miller_array_iso = get_observations_ref(iparams.hklisoin) - cc_iso_eoc, n_refl_iso_eoc = calc_cc(miller_array_iso, observations_Eoc_corrected_asu) - cc_iso_ori, n_refl_iso_ori = calc_cc(miller_array_iso, observations_original_asu) - except Exception: - dummy = 0 - #print cryst_id, shot_no, sel_type.ljust(15,' '), 'mismatch spacegroup %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f'%(a,b,c,alpha,beta,gamma) - - if cc_eoc > 0: - pickle_filename_list.append(pickle_filename) - cc_list.append(cc_eoc) - n_refl_list.append(n_refl_eoc) - - txt_out += cryst_id + ' ' + str(shot_no).zfill(iparams.iotacc.LEN_SHOT_SEQ) + \ - ' %6.2f %6.0f %6.2f %6.2f %6.0f %6.2f %6.2f %6.0f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f'%(\ - observations_original.d_min(), len(observations_original.data()), cc_ori, cc_eoc, n_refl_eoc, \ - cc_iso_ori, cc_iso_eoc, n_refl_iso_eoc, a,b,c,alpha,beta,gamma) + ' ' + pickle_filename_only + '\n' - - #select and copy best_by_cc shot - if len(pickle_filename_list) > 0: - #sort by cc and select best n results - cc_thres = flex.max(cc_list) - (flex.max(cc_list)*iparams.iotacc.percent_top_cc/100) - i_sel = cc_list > cc_thres - - cc_list_sel = cc_list.select(i_sel) - n_refl_list_sel = n_refl_list.select(i_sel) - i_seq_sel = flex.int(range(len(cc_list))).select(i_sel) - pickle_filename_list_sel = [pickle_filename_list[ii_sel] for ii_sel in i_seq_sel] - - #sort by n_refl and select the first - i_sorted = flex.sort_permutation(n_refl_list_sel, reverse=True) - n_refl_sel = n_refl_list_sel[i_sorted[0]] - cc_sel = cc_list_sel[i_sorted[0]] - pickle_filename_sel = pickle_filename_list_sel[i_sorted[0]] - - pickle_filename_sel_split = pickle_filename_sel.split('/') - pickle_filename_sel_only = pickle_filename_sel_split[len(pickle_filename_sel_split)-1] - pickle_filename_out = iparams.run_no + '/' + iparams.iotacc.set_id + '_' + pickle_filename_sel_only - - #shutil.copyfile(pickle_filename_sel, pickle_filename_out) - txt_out += 'Select --> CC_thres=%6.2f Best CC=%6.2f N_refl=%4.0f '%(cc_thres, cc_sel, n_refl_sel) + pickle_filename_out + '\n' - flag_success = True - - else: - txt_out += 'No image with CC > 0\n' - - if flag_success: - return pickle_filename_sel, txt_out - else: - return None - - -if __name__=="__main__": - iparams, txt_out_input = read_input(sys.argv[:1]) - print(txt_out_input) - - if iparams.iotacc.set_id is not None: - data_dir = iparams.data[0] + '/' +iparams.iotacc.set_id - else: - data_dir = iparams.data[0] - - cryst_ids = [] - for cryst_id in os.listdir(data_dir): - cryst_ids.append(cryst_id) - - shot_nos = list(range(iparams.iotacc.n_shots)) - run_no = iparams.run_no - - if os.path.exists(run_no): - shutil.rmtree(run_no) - - os.makedirs(run_no) - - txt_out_log = 'iotacc\n' - txt_out_log += txt_out_input - txt_out_pickle_filename_sel = '' - for cryst_id in cryst_ids: - def select_best_by_cc_mproc_wrapper(arg): - return select_best_by_cc_mproc(arg, cryst_id, iparams) - - select_best_by_cc_results = pool_map( - args=shot_nos, - func=select_best_by_cc_mproc_wrapper, - processes=iparams.n_processors) - - - for result in select_best_by_cc_results: - if result is not None: - pickle_filename_sel, txt_out = result - txt_out_log += txt_out - txt_out_pickle_filename_sel += pickle_filename_sel + '\n' - print(txt_out) - - f = open(run_no+'/pickle_selected.txt', 'w') - f.write(txt_out_pickle_filename_sel) - f.close() - - f = open(run_no+'/log.txt', 'w') - f.write(txt_out_log) - f.close() diff --git a/prime/command_line/mpi_run.py b/prime/command_line/mpi_run.py deleted file mode 100644 index 8f1639dbe87..00000000000 --- a/prime/command_line/mpi_run.py +++ /dev/null @@ -1,206 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.mpi_run -""" -Find initial scaling factors for all integration results -""" -from __future__ import absolute_import, division, print_function -from libtbx.mpi4py import MPI -import sys, os -from prime.postrefine.mod_input import process_input, read_pickles -from prime.postrefine.mod_util import intensities_scaler -from prime.postrefine.mod_merge_data import merge_data_handler -from prime.postrefine import postref_handler -from cctbx.array_family import flex -import time, math -from six.moves import range -from six.moves import zip - -# setup mpi -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() -assert size>1 - -def master(frame_token, iparams, activity): - if activity == "scale": - n_batch = 1 - frame_objects = frame_token - indices = range(0, len(frame_objects), n_batch) - for i in indices: - i_end = i+n_batch if i+n_batch < len(frame_objects) else len(frame_objects) - rankreq = comm.recv(source=MPI.ANY_SOURCE) - comm.send((activity, (frame_objects[i:i_end], iparams)), dest=rankreq) - elif activity == "pre_merge": - frame_objects, avg_mode = frame_token - n_batch = int(len(frame_objects)/(size*3)) - if n_batch < 10: n_batch = 10 - indices = range(0, len(frame_objects), n_batch) - for i in indices: - i_end = i+n_batch if i+n_batch < len(frame_objects) else len(frame_objects) - rankreq = comm.recv(source=MPI.ANY_SOURCE) - comm.send((activity, (frame_objects[i:i_end], iparams, avg_mode)), dest=rankreq) - elif activity == "merge": - frame_objects, avg_mode = frame_token - its = intensities_scaler() - cpo = its.combine_pre_merge(frame_objects, iparams) - #assign at least 100k reflections at a time - n_batch = int(1e5/(len(cpo[1])/cpo[0])) - if n_batch < 1: n_batch = 1 - print("Merging with %d batch size"%(n_batch)) - indices = range(0, cpo[0], n_batch) - for i in indices: - rankreq = comm.recv(source=MPI.ANY_SOURCE) - i_end = i+n_batch if i+n_batch < cpo[0] else cpo[0] - sel = flex.bool([sel_l and sel_h for sel_l, sel_h in zip(cpo[1]>=i, cpo[1] 1 else None - run(argv) diff --git a/prime/command_line/mpi_scale.py b/prime/command_line/mpi_scale.py deleted file mode 100644 index 8b8534ee249..00000000000 --- a/prime/command_line/mpi_scale.py +++ /dev/null @@ -1,163 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.mpi_scale -""" -Find initial scaling factors for all integration results -""" -from __future__ import absolute_import, division, print_function -from libtbx.mpi4py import MPI -import sys, os -from prime.postrefine.mod_input import process_input, read_pickles -from prime.postrefine.mod_util import intensities_scaler -from prime.postrefine.mod_merge_data import merge_data_handler -from cctbx.array_family import flex -import time, math -from six.moves import range -from six.moves import zip - -# setup mpi -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() -assert size>1 - -def master(frame_objects, iparams, activity): - if activity == "scale": - n_batch = 1 - indices = range(0, len(frame_objects), n_batch) - for i in indices: - i_end = i+n_batch if i+n_batch < len(frame_objects) else len(frame_objects) - rankreq = comm.recv(source=MPI.ANY_SOURCE) - comm.send((activity, (frame_objects[i:i_end], iparams)), dest=rankreq) - if activity == "pre_merge": - n_batch = int(len(frame_objects)/(size*3)) - if n_batch < 10: n_batch = 10 - indices = range(0, len(frame_objects), n_batch) - for i in indices: - i_end = i+n_batch if i+n_batch < len(frame_objects) else len(frame_objects) - rankreq = comm.recv(source=MPI.ANY_SOURCE) - comm.send((activity, (frame_objects[i:i_end], iparams)), dest=rankreq) - if activity == "merge": - its = intensities_scaler() - cpo = its.combine_pre_merge(frame_objects, iparams) - #assign at least 100k reflections at a time - n_batch = int(1e5/(len(cpo[1])/cpo[0])) - if n_batch < 1: n_batch = 1 - print("Merging with %d batch size"%(n_batch)) - indices = range(0, cpo[0], n_batch) - for i in indices: - rankreq = comm.recv(source=MPI.ANY_SOURCE) - i_end = i+n_batch if i+n_batch < cpo[0] else cpo[0] - sel = flex.bool([sel_l and sel_h for sel_l, sel_h in zip(cpo[1]>=i, cpo[1] 1 else None - run(argv) diff --git a/prime/command_line/plot_energy_spectrum.py b/prime/command_line/plot_energy_spectrum.py deleted file mode 100644 index b72e40d9070..00000000000 --- a/prime/command_line/plot_energy_spectrum.py +++ /dev/null @@ -1,28 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.plot_energy_spectrum -""" -Author : Uervirojnangkoorn, M. -Created : 5/16/2016 -Description : read a single energy spectrum file (csv) and plot the spectrum -""" -from __future__ import absolute_import, division, print_function -import sys -from cctbx.array_family import flex -import matplotlib.pyplot as plt - -if (__name__ == "__main__"): - if len(sys.argv)==1: - print('Usage: prime.plot_energy_spectrum energy_as_comma_separated.file') - exit() - energy_file = sys.argv[1] - - pf = open(energy_file,'r') - data = pf.read().split('\n') - x = flex.double() - y = flex.double() - for row in data: - cols = row.split(',') - if len(cols) == 2: - x.append(float(cols[1])) - y.append(float(cols[0])) - plt.plot(x,y) - plt.show() diff --git a/prime/command_line/plotstats.py b/prime/command_line/plotstats.py deleted file mode 100644 index 37c6db5502e..00000000000 --- a/prime/command_line/plotstats.py +++ /dev/null @@ -1,35 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.plotstats -''' -Author : Uervirojnangkoorn, M. -Created : 7/11/2016 -Description : Plot stats by post-refinement cycles. -''' -from __future__ import absolute_import, division, print_function -import matplotlib.pyplot as plt -import sys -from six.moves import cPickle as pickle -from six.moves import range - -if len(sys.argv)==1: - print('Use prime.plotstats to view different stats along post-refinement cycles.') - print('Usage: prime.plotstats path/to/run/folder') - exit() -run_no = sys.argv[1] -stat_pickle = pickle.load(open(run_no+"/pickle.stat","rb")) -total_i_o_sigi = stat_pickle['total_i_o_sigi'] -total_completeness = stat_pickle['total_completeness'] -total_rmerge = stat_pickle['total_rmerge'] -total_n_obs = stat_pickle['total_n_obs'] -total_cc12 = stat_pickle['total_cc12'] -x = list(range(len(total_cc12))) -#plot -plt.subplot(1,2,1, title='CC1/2') -plt.plot(x, total_cc12, linewidth=2.0) -plt.grid(True) -plt.ylim([min(total_cc12)-5,100]) -plt.xlabel('No. of cycles') -plt.subplot(1,2,2, title='Rmerge') -plt.plot(x, total_rmerge, linewidth=2.0) -plt.grid(True) -plt.xlabel('No. of cycles') -plt.show() diff --git a/prime/command_line/postrefine.py b/prime/command_line/postrefine.py deleted file mode 100644 index 92fa0a980dc..00000000000 --- a/prime/command_line/postrefine.py +++ /dev/null @@ -1,209 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.postrefine -''' -Author : Uervirojnangkoorn, M. -Created : 7/13/2014 -Description : Main commandline for prime. -''' -from __future__ import absolute_import, division, print_function - -from libtbx.easy_mp import parallel_map -from cctbx.array_family import flex -from prime.command_line.solve_indexing_ambiguity import indexing_ambiguity_handler -from prime.postrefine.mod_mx import mx_handler -from prime.postrefine.mod_input import read_pickles -from prime.postrefine.mod_util import intensities_scaler -import os, sys, math -import numpy as np -from datetime import datetime, time -from libtbx.utils import Usage -from six.moves import range -from six.moves import zip - -def determine_mean_I_mproc(args): - from prime.postrefine import postref_handler - prh = postref_handler() - frame_file, iparams, avg_mode = args - mean_I = prh.calc_mean_intensity(frame_file, iparams, avg_mode) - return mean_I - -def scale_frame_by_mean_I_mproc(args): - from prime.postrefine import postref_handler - prh = postref_handler() - frame_no, frame_file, iparams, mean_of_mean_I, avg_mode = args - pres = prh.scale_frame_by_mean_I(frame_no, frame_file, iparams, mean_of_mean_I, avg_mode) - return pres - -def postrefine_by_frame_mproc(args): - from prime.postrefine import postref_handler - prh = postref_handler() - frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode = args - pres = prh.postrefine_by_frame(frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode) - return pres - -def scale_frames(frames, frame_files, iparams): - """scale frames""" - avg_mode = 'average' - if iparams.flag_apply_b_by_frame: - mean_of_mean_I = 0 - else: - #Calculate for each frame - frame_args = [(frame_file, iparams, avg_mode) for frame_file in frame_files] - determine_mean_I_result = parallel_map( - iterable=frame_args, - func=determine_mean_I_mproc, - processes=iparams.n_processors) - frames_mean_I = flex.double() - for result in determine_mean_I_result: - if result is not None: - mean_I, txt_out_result = result - if mean_I is not None: - frames_mean_I.append(mean_I) - mean_of_mean_I = np.median(frames_mean_I) - #use the calculate to scale each frame - frame_args = [(frame_no, frame_file, iparams, mean_of_mean_I, avg_mode) for frame_no, frame_file in zip(frames, frame_files)] - scale_frame_by_mean_I_result = parallel_map( - iterable=frame_args, - func=scale_frame_by_mean_I_mproc, - processes=iparams.n_processors) - observations_merge_mean_set = [] - for result in scale_frame_by_mean_I_result: - if result is not None: - pres, txt_out_result = result - if pres is not None: - observations_merge_mean_set.append(pres) - return observations_merge_mean_set - -def merge_frames(pres_set, iparams, avg_mode='average', mtz_out_prefix='mean_scaled'): - """merge frames using average as the default""" - miller_array_ref, txt_out = (None,'') - intscal = intensities_scaler() - if pres_set: - prep_output = intscal.prepare_output(pres_set, iparams, avg_mode) - if prep_output: - mdh, _, txt_out_rejection = intscal.calc_avg_I_cpp(prep_output, iparams, avg_mode) - #select only indices with non-Inf non-Nan stats - selections = flex.bool([False if (math.isnan(r0) or math.isinf(r0) or math.isnan(r1) or math.isinf(r1)) else True for r0, r1 in zip(mdh.r_meas_div, mdh.r_meas_divisor)]) - mdh.reduce_by_selection(selections) - - #handle rejected reflections - rejections = {} - for reject in txt_out_rejection.split('\n'): - data = reject.split() - if data: - if not data[0] in rejections: - rejections[data[0]] = flex.miller_index() - rejections[data[0]].append(tuple([int(_d) for _d in data[1:4]])) - - if len(rejections) > 0: - if not iparams.rejections: - iparams.rejections = {} - iparams.rejections.update(rejections) - - #merge all good indices - mdh, txt_merge_mean_table = intscal.write_output(mdh, - iparams, iparams.run_no+'/'+mtz_out_prefix, avg_mode) - print(txt_merge_mean_table) - print(prep_output[-1]) - txt_out = txt_merge_mean_table + prep_output[-1] - return mdh, txt_out - -def postrefine_frames(i_iter, frames, frame_files, iparams, pres_set, miller_array_ref, avg_mode): - """postrefine given frames and previous postrefinement results""" - miller_array_ref = miller_array_ref.generate_bijvoet_mates() - txt_merge_postref = 'Post-refinement cycle '+str(i_iter+1)+' ('+avg_mode+')\n' - txt_merge_postref += ' * R and CC show percent change.\n' - print(txt_merge_postref) - frame_args = [(frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode) for frame_no, frame_file, pres_in in zip(frames, frame_files, pres_set)] - postrefine_by_frame_result = parallel_map( - iterable=frame_args, - func=postrefine_by_frame_mproc, - processes=iparams.n_processors) - postrefine_by_frame_good = [] - postrefine_by_frame_pres_list = [] - for results in postrefine_by_frame_result: - if results is not None: - pres, txt_out_result = results - postrefine_by_frame_pres_list.append(pres) - if pres is not None: - postrefine_by_frame_good.append(pres) - else: - postrefine_by_frame_pres_list.append(None) - return postrefine_by_frame_good, postrefine_by_frame_pres_list, txt_merge_postref - -def run(argv): - #capture starting time - time_global_start=datetime.now() - import logging - logging.captureWarnings(True) - formatter = logging.Formatter('%(asctime)s\t%(levelname)s\t%(message)s') - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.ERROR) - console_handler.setFormatter(formatter) - logging.getLogger().addHandler(console_handler) - logging.getLogger('py.warnings').addHandler(console_handler) - logging.basicConfig(format='%(asctime)s\t%(levelname)s\t%(message)s', level=logging.DEBUG) - #0.1 determine indexing ambiguity and setup iparams - txt_indexing_ambiguity = "Determine if there is an indexing ambiguity on the dataset" - print(txt_indexing_ambiguity) - idah = indexing_ambiguity_handler() - sol_pickle, iparams = idah.run(argv) - if sol_pickle is None: - print("No ambiguity.") - txt_indexing_ambiguity += "\nNo ambiguity." - else: - print("Ambiguity is solved. Solution file was saved to result folder.") - txt_indexing_ambiguity += "Ambiguity is solved. Solution file was saved to result folder." - iparams.indexing_ambiguity.index_basis_in = sol_pickle - #0.2 setup parameters - iparams.flag_volume_correction = False - if iparams.partiality_model == "Lognormal": - iparams.voigt_nu = 0.008 #use voigt_nu as lognpdf zero parameter - #0.3 read frames - frame_files = read_pickles(iparams.data) - frames = range(len(frame_files)) - #1. prepare reference miller array - txt_merge_mean = 'Generating a reference set (will not be used if hklrefin is set)' - print(txt_merge_mean) - #Always generate the mean-intensity scaled set. - scaled_pres_set = scale_frames(frames, frame_files, iparams) - mdh, _txt_merge_mean = merge_frames(scaled_pres_set, iparams) - miller_array_ref = mdh.miller_array_merge - txt_merge_mean += '\n' + _txt_merge_mean - if not iparams.n_postref_cycle: - with open(iparams.run_no+'/log.txt', 'a') as f: - f.write(txt_indexing_ambiguity + txt_merge_mean) - raise Usage("No. of post-refinement cycle was set to 0. Exit without post-refinement.") - if iparams.hklrefin is not None: - mxh = mx_handler() - _, miller_array_ref = mxh.get_miller_array_from_reflection_file(iparams.hklrefin) - if miller_array_ref is None: - raise Usage("Problem with the assigned reference set. Try setting hklrefin=None and rerun the program.") - #2. Post-refinement - txt_merge_postref = '' - postref_pres_set = [None]*len(frames) - avg_mode = 'weighted' - for i_iter in range(iparams.n_postref_cycle): - if i_iter == (iparams.n_postref_cycle-1): avg_mode = 'final' - postref_good_pres_set, postref_pres_set, _txt_merge_postref = postrefine_frames(i_iter, frames, frame_files, iparams, postref_pres_set, miller_array_ref, avg_mode) - if postref_good_pres_set: - mdh, _txt_merge_postref = merge_frames(postref_good_pres_set, iparams, - avg_mode=avg_mode, mtz_out_prefix='postref_cycle_'+str(i_iter+1)) - miller_array_ref = mdh.miller_array_merge - txt_merge_postref += _txt_merge_postref - else: - raise Usage("Problem with post-refinement. No images refined. Please check your input file.") - #3. collect caculating time - time_global_end=datetime.now() - time_global_spent=time_global_end-time_global_start - txt_out_time_spent = 'Total calculation time: '+'{0:.2f}'.format(time_global_spent.seconds)+ \ - ' seconds\nFinished: '+time_global_end.strftime("%A %d. %B %Y %H:%M:%S")+'\n' - print(txt_out_time_spent) - txt_out = txt_indexing_ambiguity + txt_merge_mean + txt_merge_postref + txt_out_time_spent - with open(os.path.join(iparams.run_no,'log.txt'), 'a') as f: - f.write(txt_out) - with open(os.path.join(iparams.run_no,'.done'), 'w') as f: - f.write('Done') - return mdh - -if (__name__ == "__main__"): - run(sys.argv[1:] if len(sys.argv) > 1 else None) diff --git a/prime/command_line/prime_gui.py b/prime/command_line/prime_gui.py deleted file mode 100644 index 1fcee69152e..00000000000 --- a/prime/command_line/prime_gui.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import absolute_import, division, print_function - -# LIBTBX_SET_DISPATCHER_NAME prime -# LIBTBX_PRE_DISPATCHER_INCLUDE_SH export PHENIX_GUI_ENVIRONMENT=1 -# LIBTBX_PRE_DISPATCHER_INCLUDE_SH export BOOST_ADAPTBX_FPE_DEFAULT=1 - -''' -Author : Lyubimov, A.Y. -Created : 05/19/2016 -Last Changed: 06/07/2019 -Description : PRIME GUI startup module. -''' - -import wx -from prime.postrefine.mod_gui_frames import PRIMEWindow - -class MainApp(wx.App): - """ App to launch the main GUI window """ - - def OnInit(self): - from platform import python_version - import matplotlib - - print('Library Versions:') - print(' Python : ', python_version()) - print(' wxPython : ', wx.__version__) - print(' MatPlotLib : ', matplotlib.__version__) - - # Initialize Main window - self.frame = PRIMEWindow(None, -1, title='PRIME') - self.frame.place_and_size(set_size='v_default', set_by='mouse', center=True) - - # Show main window - self.frame.Show(True) - - self.SetTopWindow(self.frame) - return True - - -if __name__ == '__main__': - app = MainApp() - app.MainLoop() diff --git a/prime/command_line/print_integration_pickle.py b/prime/command_line/print_integration_pickle.py deleted file mode 100644 index 5d5d02c37e5..00000000000 --- a/prime/command_line/print_integration_pickle.py +++ /dev/null @@ -1,485 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.print_integration_pickle -""" -Author : Uervirojnangkoorn, M. -Created : 11/1/2015 -Description : read integration pickles and view systemetic absences and beam X, Y position -""" -from __future__ import absolute_import, division, print_function - -from six.moves import cPickle as pickle -from cctbx.array_family import flex -from iotbx import reflection_file_reader -import sys, math -from scitbx.matrix import sqr -import numpy as np -import matplotlib.pyplot as plt -import matplotlib.mlab as mlab -from cctbx.uctbx import unit_cell -from prime.postrefine.mod_leastsqr import good_unit_cell -from cctbx import statistics -from prime.postrefine.mod_input import read_frame, read_pickles -from six.moves import range -from six.moves import zip - -def calc_wilson(observations_full, n_residues): - """ - Caculate isotropic Wilson G and B-factors - """ - if n_residues == 0: - return 0, 0 - from prime.postrefine.mod_util import mx_handler - mxh = mx_handler() - asu_contents = mxh.get_asu_contents(n_residues) - try: - observations_as_f = observations_full.as_amplitude_array() - binner_template_asu = observations_as_f.setup_binner(auto_binning=True) - wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) - G = wp.wilson_intensity_scale_factor - B = wp.wilson_b - except Exception: - G,B = (0,0) - return G, B - -def get_miller_array_from_mtz(mtz_filename): - flag_hklisoin_found = False - miller_array_iso = None - if mtz_filename is not None: - flag_hklisoin_found = True - reflection_file_iso = reflection_file_reader.any_reflection_file(mtz_filename) - miller_arrays_iso=reflection_file_iso.as_miller_arrays() - is_found_iso_as_intensity_array = False - is_found_iso_as_amplitude_array = False - for miller_array in miller_arrays_iso: - if miller_array.is_xray_intensity_array(): - miller_array_iso = miller_array.deep_copy() - is_found_iso_as_intensity_array = True - break - elif miller_array.is_xray_amplitude_array(): - is_found_iso_as_amplitude_array = True - miller_array_converted_to_intensity = miller_array.as_intensity_array() - if is_found_iso_as_intensity_array == False: - if is_found_iso_as_amplitude_array: - miller_array_iso = miller_array_converted_to_intensity.deep_copy() - else: - flag_hklisoin_found = False - if miller_array_iso is not None: - perm = miller_array_iso.sort_permutation(by_value="resolution", reverse=True) - miller_array_iso = miller_array_iso.select(perm) - return flag_hklisoin_found, miller_array_iso - -def read_input(args): - if len(args) == 0: - print("prime.print_integration_pickle: for viewing systematic absences and beam xy position from integration pickles.") - print("usage: prime.print_integration_pickle data=integrated.lst pixel_size_mm=0.079346 check_sys_absent=True target_space_group=P212121") - exit() - data = [] - hklrefin = None - pixel_size_mm = None - check_sys_absent = False - target_space_group = None - target_unit_cell = None - target_anomalous_flag = False - flag_plot = True - d_min = 0 - d_max = 99 - n_residues = 0 - for i in range(len(args)): - pair=args[i].split('=') - if pair[0]=='data': - data.append(pair[1]) - if pair[0]=='hklrefin': - hklrefin = pair[1] - if pair[0]=='pixel_size_mm': - pixel_size_mm = float(pair[1]) - if pair[0]=='check_sys_absent': - check_sys_absent = bool(pair[1]) - if pair[0]=='target_space_group': - target_space_group = pair[1] - if pair[0]=='target_unit_cell': - try: - tuc = pair[1].split(',') - a,b,c,alpha,beta,gamma = (float(tuc[0]), float(tuc[1]), float(tuc[2]), \ - float(tuc[3]), float(tuc[4]), float(tuc[5])) - target_unit_cell = unit_cell((a,b,c,alpha,beta,gamma)) - except Exception: - pass - if pair[0]=='target_anomalous_flag': - target_anomalous_flag = bool(pair[1]) - if pair[0]=='flag_plot': - if pair[1] == 'False': - flag_plot = False - if pair[0]=='d_min': - d_min = float(pair[1]) - if pair[0]=='d_max': - d_max = float(pair[1]) - if pair[0]=='n_residues': - n_residues = int(pair[1]) - if len(data)==0: - print("Please provide data path. (eg. data=/path/to/pickle/)") - exit() - if check_sys_absent: - if target_space_group is None: - print("Please provide target space group if you want to check systematic absence (eg. target_space_group=P212121)") - exit() - if pixel_size_mm is None: - print("Please specify pixel size (eg. pixel_size_mm=0.079346)") - exit() - return data, hklrefin, pixel_size_mm, check_sys_absent, target_unit_cell, target_space_group, target_anomalous_flag, flag_plot, d_min, d_max, n_residues - - -if (__name__ == "__main__"): - cc_bin_low_thres = 0.25 - beam_thres = 0.5 - uc_tol = 20 - #0 .read input parameters and frames (pickle files) - data, hklrefin, pixel_size_mm, check_sys_absent_input, target_unit_cell, \ - target_space_group, target_anomalous_flag, flag_plot, d_min, d_max, n_residues = read_input(args = sys.argv[1:]) - frame_files = read_pickles(data) - xbeam_set = flex.double() - ybeam_set = flex.double() - sys_abs_set = [] - sys_abs_all = flex.double() - cc_bin_low_set = flex.double() - cc_bins_set = [] - d_bins_set = [] - oodsqr_bins_set = [] - flag_good_unit_cell_set = [] - print('Summary of integration pickles:') - print('(image file, min. res., max. res, beamx, beamy, n_refl, cciso, , a, b, c, mosaicity, residual, dd_mm, wavelength, good_cell?, G, B)') - uc_a = flex.double() - uc_b = flex.double() - uc_c = flex.double() - dd_mm = flex.double() - wavelength_set = flex.double() - for pickle_filename in frame_files: - check_sys_absent = check_sys_absent_input - observations_pickle = read_frame(pickle_filename) - pickle_filename_arr = pickle_filename.split('/') - pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr)-1] - flag_hklisoin_found, miller_array_iso = get_miller_array_from_mtz(hklrefin) - observations = observations_pickle["observations"][0] - swap_dict = {'test_xx1.pickle':0, \ - 'tes_xx2.pickle':0} - if pickle_filename_only in swap_dict: - from cctbx import sgtbx - cb_op = sgtbx.change_of_basis_op('a,c,b') - observations = observations.change_basis(cb_op) - #from cctbx import sgtbx - #cb_op = sgtbx.change_of_basis_op('c,b,a') - #observations = observations.change_basis(cb_op) - #apply constrain using the crystal system - #check systematic absent - if check_sys_absent: - try: - from cctbx.crystal import symmetry - miller_set = symmetry( - unit_cell=observations.unit_cell().parameters(), - space_group_symbol=target_space_group - ).build_miller_set( - anomalous_flag=target_anomalous_flag, - d_min=observations.d_min()) - observations = observations.customized_copy(anomalous_flag=target_anomalous_flag, - crystal_symmetry=miller_set.crystal_symmetry()) - except Exception: - print('Cannot apply target space group: observed space group=', observations.space_group_info()) - check_sys_absent = False - #check if the uc is good - flag_good_unit_cell = False - if check_sys_absent: - if target_unit_cell is None: - flag_good_unit_cell = True - else: - flag_good_unit_cell = good_unit_cell(observations.unit_cell().parameters(), None, uc_tol, target_unit_cell=target_unit_cell) - flag_good_unit_cell_set.append(flag_good_unit_cell) - #calculate partiality - wavelength = observations_pickle["wavelength"] - crystal_init_orientation = observations_pickle["current_orientation"][0] - detector_distance_mm = observations_pickle['distance'] - mm_predictions = pixel_size_mm*(observations_pickle['mapped_predictions'][0]) - xbeam = observations_pickle["xbeam"] - ybeam = observations_pickle["ybeam"] - xbeam_set.append(xbeam) - ybeam_set.append(ybeam) - alpha_angle = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ - for pred in mm_predictions]) - spot_pred_x_mm = flex.double([pred[0]-xbeam for pred in mm_predictions]) - spot_pred_y_mm = flex.double([pred[1]-ybeam for pred in mm_predictions]) - #calculate mean unit cell - if flag_good_unit_cell: - uc_a.append(observations.unit_cell().parameters()[0]) - uc_b.append(observations.unit_cell().parameters()[1]) - uc_c.append(observations.unit_cell().parameters()[2]) - dd_mm.append(detector_distance_mm) - wavelength_set.append(wavelength) - #resoultion filter - i_sel_res = observations.resolution_filter_selection(d_min=d_min, d_max=d_max) - observations = observations.select(i_sel_res) - alpha_angle = alpha_angle.select(i_sel_res) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res) - #sort by resolution - perm = observations.sort_permutation(by_value="resolution", reverse=True) - observations = observations.select(perm) - alpha_angle = alpha_angle.select(perm) - spot_pred_x_mm = spot_pred_x_mm.select(perm) - spot_pred_y_mm = spot_pred_y_mm.select(perm) - from prime.postrefine.mod_partiality import partiality_handler - ph = partiality_handler() - r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), - observations.indices(), wavelength) - two_theta = observations.two_theta(wavelength=wavelength).data() - sin_theta_over_lambda_sq = observations.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() - ry, rz, re, nu, rotx, roty = (0, 0, 0.003,0.5, 0, 0) - flag_beam_divergence = False - partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(crystal_init_orientation.unit_cell(), - rotx, roty, observations.indices(), - ry, rz, r0, re, nu, two_theta, alpha_angle, wavelength, - crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, - detector_distance_mm, "Lorentzian", - flag_beam_divergence) - I_full = observations.data()/ partiality_init - sigI_full = observations.sigmas()/ partiality_init - observations_full = observations.customized_copy(data=I_full, sigmas=sigI_full) - wilson_G, wilson_B = calc_wilson(observations_full, n_residues) - #calculate R and cc with reference - cc_iso, cc_full_iso, cc_bin_low, cc_bin_med = (0, 0, 0, 0) - observations_asu = observations.map_to_asu() - observations_full_asu = observations_full.map_to_asu() - cc_bins = flex.double() - oodsqr_bins = flex.double() - d_bins = flex.double() - if flag_hklisoin_found: - #build observations dict - obs_dict = {} - for mi_asu, I, sigI, I_full, sigI_full in zip(observations_asu.indices(), \ - observations_asu.data(), observations_asu.sigmas(), \ - observations_full_asu.data(), observations_full_asu.sigmas()): - obs_dict[mi_asu]=(I, sigI, I_full, sigI_full) - I_match = flex.double() - I_full_match = flex.double() - I_iso_match = flex.double() - d_match = flex.double() - oodsqr_match = flex.double() - for mi_asu, d, I_iso in zip(miller_array_iso.indices(), miller_array_iso.d_spacings().data(), miller_array_iso.data()): - if mi_asu in obs_dict: - I, sigI, I_full, sigI_full = obs_dict[mi_asu] - I_match.append(I) - I_full_match.append(I_full) - I_iso_match.append(I_iso) - oodsqr_match.append(1/(d**2)) - d_match.append(d) - #calculate correlation - try: - cc_iso = np.corrcoef(I_iso_match, I_match)[0,1] - cc_full_iso = np.corrcoef(I_iso_match, I_full_match)[0,1] - except Exception: - pass - #scatter plot - if flag_plot and len(frame_files)==1: - plt.subplot(211) - #plt.scatter(oodsqr_match, I_iso_match, s=10, marker='o', c='r') - plt.plot(oodsqr_match, I_iso_match) - plt.title('Reference intensity') - plt.xlabel('1/d^2') - plt.ylabel('I_ref') - plt.subplot(212) - #plt.scatter(oodsqr_match, I_match, s=10, marker='o', c='r') - plt.plot(oodsqr_match, I_match) - plt.title('Observed intensity CC=%.4g'%(cc_iso)) - plt.xlabel('1/d^2') - plt.ylabel('I_obs') - plt.show() - #scatter bin plot - n_bins = 10 - n_refl = int(round(len(I_match)/n_bins)) - if len(I_match) > 0: - for i_bin in range(n_bins): - i_start = i_bin*n_refl - if i_bin < n_bins - 1: - i_end = (i_bin*n_refl) + n_refl - else: - i_end = -1 - I_iso_bin = I_iso_match[i_start:i_end] - I_bin = I_match[i_start:i_end] - d_bin = d_match[i_start:i_end] - try: - cc_bin = np.corrcoef(I_iso_bin, I_bin)[0,1] - except Exception: - cc_bin = 0 - cc_bins.append(cc_bin) - try: - min_d_bin = np.min(d_bin) - except Exception: - min_d_bin = 1 - d_bins.append(min_d_bin) - oodsqr_bins.append(1/(min_d_bin**2)) - if i_bin == 0: - cc_bin_low = cc_bin - if i_bin == 5: - cc_bin_med = cc_bin - if flag_plot and len(frame_files)==1: - plt.subplot(2,5,i_bin+1) - plt.scatter(I_iso_bin, I_bin,s=10, marker='o', c='r') - plt.title('Bin %2.0f (%6.2f-%6.2f A) CC=%6.2f'%(i_bin+1, np.max(d_bin), np.min(d_bin), cc_bin)) - if i_bin == 0: - plt.xlabel('I_ref') - plt.ylabel('I_obs') - if flag_plot and len(frame_files)==1: - plt.show() - #print full detail if given a single file - if len(frame_files) == 1: - print('Crystal orientation') - print(crystal_init_orientation.crystal_rotation_matrix()) - print('Direct matrix') - print(crystal_init_orientation.direct_matrix()) - a, b, c, alpha, beta, gamma = observations.unit_cell().parameters() - txt_out_head= '{0:40} {1:5.2f} {2:5.2f} {3:5.2f} {4:5.2f} {5:5.0f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f} {15:6.2f} {16:6.2f} {20:6.4f} {17:5} {18:6.2f} {19:6.2f}'.format(pickle_filename_only + ' ('+ str(observations.crystal_symmetry().space_group_info())+')', observations.d_min(), np.max(observations.d_spacings().data()), xbeam, ybeam, len(observations.data()), cc_iso, np.mean(cc_bins), a, b, c, alpha, beta, gamma, observations_pickle["mosaicity"], observations_pickle["residual"], detector_distance_mm, flag_good_unit_cell, wilson_G, wilson_B, wavelength) - print(txt_out_head) - cc_bin_low_set.append(cc_iso) - cc_bins_set.append(cc_bins) - d_bins_set.append(d_bins) - oodsqr_bins_set.append(oodsqr_bins) - sys_abs_lst = flex.double() - if check_sys_absent: - cn_refl = 0 - for sys_absent_flag, d_spacing, miller_index_ori, miller_index_asu, I, sigI in zip(observations.sys_absent_flags(), observations.d_spacings().data(), observations.indices(), observations_asu.indices(), observations.data(), observations.sigmas()): - if sys_absent_flag[1]: - txt_out = '{9:40} {10:6.2f} {0:3} {1:3} {2:3} {3:3} {4:3} {5:3} {6:8.2f} {7:8.2f} {8:6.2f}'.format(miller_index_ori[0], miller_index_ori[1], miller_index_ori[2], miller_index_asu[0], miller_index_asu[1], miller_index_asu[2], I, sigI, I/sigI, pickle_filename_only, d_spacing) - #if I/sigI > 1.5: - # print txt_out - print(txt_out) - cn_refl +=1 - sys_abs_lst.append(I/sigI) - sys_abs_all.append(I/sigI) - sys_abs_set.append(sys_abs_lst) - #collect beamxy - xbeam_mean = flex.mean(xbeam_set) - xbeam_std = np.std(xbeam_set) - ybeam_mean = flex.mean(ybeam_set) - ybeam_std = np.std(ybeam_set) - xbeam_filtered_set = flex.double() - ybeam_filtered_set = flex.double() - frame_filtered_set = [] - sys_abs_all_filtered = flex.double() - txt_out = '' - txt_out_mix = '' - txt_out_uc = '' - txt_out_report_beam_filter = 'Images with beam center displaced > %6.2f mm.:\n'%(beam_thres) - txt_out_report_cc_filter = 'Images with cc < %6.2f:\n'%(cc_bin_low_thres) - from scitbx.matrix import col - for pickle_filename, xbeam, ybeam, sys_abs_lst, cc_bin_low, flag_good_unit_cell in \ - zip(frame_files, xbeam_set, \ - ybeam_set, sys_abs_set, cc_bin_low_set, flag_good_unit_cell_set): - pickle_filename_arr = pickle_filename.split('/') - pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr)-1] - pred_xy = col((xbeam, ybeam)) - calc_xy = col((xbeam_mean, ybeam_mean)) - diff_xy = pred_xy - calc_xy - txt_out_report_tmp = '{0:80} {1:6.2f} {2:6.2f} {3:6.2f} {4:6.4f}\n'.format(pickle_filename_only, xbeam, ybeam, cc_bin_low, diff_xy.length()) - if diff_xy.length() < beam_thres: - xbeam_filtered_set.append(xbeam) - ybeam_filtered_set.append(ybeam) - frame_filtered_set.append(pickle_filename) - txt_out += pickle_filename + '\n' - sys_abs_all_filtered.extend(sys_abs_lst) - if cc_bin_low > cc_bin_low_thres and flag_good_unit_cell: - txt_out_mix += pickle_filename + '\n' - else: - txt_out_report_cc_filter += txt_out_report_tmp - if flag_good_unit_cell: - txt_out_uc += pickle_filename + '\n' - else: - txt_out_report_beam_filter += txt_out_report_tmp - print() - print('CC mean=%6.2f median=%6.2f std=%6.2f'%(flex.mean(cc_bin_low_set), np.median(cc_bin_low_set), np.std(cc_bin_low_set))) - print('Xbeam mean=%8.4f std=%6.4f'%(xbeam_mean, xbeam_std)) - print('Ybeam mean=%8.4f std=%6.4f'%(ybeam_mean, ybeam_std)) - print('UC mean a=%8.4f (%8.4f) b=%8.4f (%8.4f) c=%9.4f (%8.4f)'%(flex.mean(uc_a), np.std(uc_a), flex.mean(uc_b), np.std(uc_b), flex.mean(uc_c), np.std(uc_c))) - print('Detector distance mean=%8.4f (%8.4f)'%(flex.mean(dd_mm), np.std(dd_mm))) - print('Wavelength mean=%8.4f (%8.4f)'%(flex.mean(wavelength_set), np.std(wavelength_set))) - print('No. of frames: All = %6.0f Beam outliers = %6.0f CC filter=%6.0f'%(len(frame_files), len(frame_files) - (len(txt_out.split('\n'))-1), len(frame_files) - (len(txt_out_mix.split('\n'))-1))) - print() - print('Reporting outliers (image name, xbeam, ybeam, cciso, delta_xy)') - print(txt_out_report_beam_filter) - print(txt_out_report_cc_filter) - #write out filtered beamxy pickle files - f = open('integration_pickle_beam_filter.lst', 'w') - f.write(txt_out) - f.close() - #write out mix filter pickle files - f = open('integration_mix_filter.lst', 'w') - f.write(txt_out_mix) - f.close() - #write out filtered beamxy and uc pickle files - f = open('integration_pickle_beam_uc_filter.lst', 'w') - f.write(txt_out_uc) - f.close() - if flag_plot: - plt.subplot(211) - plt.plot(xbeam_set,ybeam_set,"r.") - plt.xlim([xbeam_mean-2, xbeam_mean+2]) - plt.ylim([ybeam_mean-2, ybeam_mean+2]) - #plt.axes().set_aspect("equal") - plt.title('Raw data') - plt.grid(True) - plt.subplot(212) - plt.plot(xbeam_filtered_set,ybeam_filtered_set,"r.") - plt.xlim([xbeam_mean-2, xbeam_mean+2]) - plt.ylim([ybeam_mean-2, ybeam_mean+2]) - #plt.axes().set_aspect("equal") - plt.title('After beamxy position filter') - plt.grid(True) - plt.show() - #plot I/sigI histogram for systematic absences - if len(sys_abs_set) > 0: - plt.subplot(211) - sys_abs_sel = sys_abs_all.select(flex.abs(sys_abs_all)>2) - x = sys_abs_sel.as_numpy_array() - mu = np.mean(x) - med = np.median(x) - sigma = np.std(x) - num_bins = 25 - n, bins, patches = plt.hist(x, num_bins, normed=False, facecolor='blue', alpha=0.5) - #y = mlab.normpdf(bins, mu, sigma) - #plt.plot(bins, y, 'r--') - #plt.ylim([0,70]) - #plt.xlim([-150,500]) - plt.ylabel('Frequencies') - plt.grid() - plt.title('Systematic absences with |I/sigI| > 2.0\nmean %5.3f median %5.3f sigma %5.3f' %(mu, med, sigma)) - plt.subplot(212) - sys_abs_all_filtered_sel = sys_abs_all_filtered.select(flex.abs(sys_abs_all_filtered)>2) - x = sys_abs_all_filtered_sel.as_numpy_array() - mu = np.mean(x) - med = np.median(x) - sigma = np.std(x) - num_bins = 25 - n, bins, patches = plt.hist(x, num_bins, normed=False, facecolor='blue', alpha=0.5) - #y = mlab.normpdf(bins, mu, sigma) - #plt.plot(bins, y, 'r--') - #plt.ylim([0,70]) - #plt.xlim([-150,500]) - plt.ylabel('Frequencies') - plt.grid() - plt.title('Systematic absences with |I/sigI| > 2.0 (After BeamXY filter)\nmean %5.3f median %5.3f sigma %5.3f' %(mu, med, sigma)) - plt.show() - cn_i = 0 - for cc_bins, d_bins, oodsqrt_bins, cc_bin_low, pickle_filename in zip(cc_bins_set, d_bins_set, oodsqr_bins_set, cc_bin_low_set, frame_files): - pickle_filename_arr = pickle_filename.split('/') - pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr)-1] - if cc_bin_low < cc_bin_low_thres: - plt.subplot(2,1,1) - plt.plot(oodsqrt_bins, cc_bins) - plt.title('CC by resolutions for cc < %6.2f'%(cc_bin_low_thres)) - plt.xlabel('1/d^2') - plt.ylim([0,1]) - plt.grid(True) - else: - plt.subplot(2,1,2) - plt.plot(oodsqrt_bins, cc_bins) - plt.title('CC by resolutions for cc > %6.2f'%(cc_bin_low_thres)) - plt.xlabel('1/d^2') - plt.ylim([0,1]) - plt.grid(True) - cn_i += 1 - plt.show() diff --git a/prime/command_line/r_mtz_mtz.py b/prime/command_line/r_mtz_mtz.py deleted file mode 100644 index 0cd9218a455..00000000000 --- a/prime/command_line/r_mtz_mtz.py +++ /dev/null @@ -1,101 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.r_mtz_mtz -''' -Author : Uervirojnangkoorn, M. -Created : 6/29/2016 -Description : Scale second mtz to the first (linear scale only) and calculate r-factors. -Note that all intensity array will be converted to amplitude. -''' -from __future__ import absolute_import, division, print_function - -from iotbx import reflection_file_reader -import sys -from six.moves import range - -def read_input(args): - hkla = None - hklb = '' - d_min = 0 - d_max = 99 - n_bins = 20 - flag_b = False - for i in range(len(args)): - pair=args[i].split('=') - if pair[0]=='hkla': - hkla = pair[1] - elif pair[0]=='hklb': - hklb = pair[1] - elif pair[0]=='d_min': - d_min = float(pair[1]) - elif pair[0]=='d_max': - d_max = float(pair[1]) - elif pair[0]=='n_bins': - n_bins = int(pair[1]) - elif pair[0]=='flag_b': - flag_b = bool(pair[1]) - if hklb == '': - print("Please provide input hkl files.") - exit() - return hkla, hklb, d_min, d_max, n_bins, flag_b - -def get_miller_f_from_reflection_file(hklin): - flag_hklin_found = False - miller_array_f = None - if hklin is not None: - flag_hklin_found = True - reflection_file = reflection_file_reader.any_reflection_file(hklin) - miller_arrays = reflection_file.as_miller_arrays() - is_found_as_intensity_array = False - is_found_as_amplitude_array = False - is_found_as_complex_array = False - for miller_array in miller_arrays: - if miller_array.is_xray_amplitude_array(): - miller_array_f = miller_array.deep_copy() - is_found_as_amplitude_array = True - break - elif miller_array.is_xray_intensity_array(): - is_found_as_intensity_array = True - miller_array_converted_to_amplitude = miller_array.enforce_positive_amplitudes() - elif miller_array.is_complex_array(): - is_found_as_complex_array = True - miller_array_converted_to_amplitude = miller_array.amplitudes() - if is_found_as_amplitude_array == False: - if is_found_as_intensity_array or is_found_as_complex_array: - miller_array_f = miller_array_converted_to_amplitude.deep_copy() - else: - flag_hklin_found = False - return flag_hklin_found, miller_array_f - - -if (__name__ == "__main__"): - #check input - if len(sys.argv)==1: - print('Use prime.r_mtz_mtz to calculate R-factor between two reflection files.') - print('Only linear scale will be performed. Set flag_b=True to include B-factor scaling.') - print('Usage: prime.r_mtz_mtz hkla=reflection1.mtz hklb=reflection2.mtz d_min=min_resolution d_max=max_resolution n_bins=no_of_bins flag_b=True_or_False.') - exit() - #read input parameters and frames (pickle files) - hkla, hklb, d_min, d_max, n_bins, flag_b = read_input(args = sys.argv[1:]) - #first reflection file (reference) - flag_hkla_found, miller_array_a = get_miller_f_from_reflection_file(hkla) - #second reflection file - flag_hklb_found, miller_array_b = get_miller_f_from_reflection_file(hklb) - if flag_hkla_found and flag_hklb_found: - #filter resolution - miller_array_a = miller_array_a.resolution_filter(d_min=d_min, d_max=d_max) - miller_array_b = miller_array_b.resolution_filter(d_min=d_min, d_max=d_max) - #report - print('First reflection file:', hkla) - miller_array_a.show_summary() - print('Second reflection file:', hklb) - miller_array_b.show_summary() - #scale b to a - miller_array_b_scaled = miller_array_a.scale(miller_array_b, resolution_dependent=flag_b) - #get common sets - ma_common_a, ma_common_b = miller_array_a.common_sets(miller_array_b_scaled) - #set up binning - ma_common_a.setup_binner(n_bins=n_bins) - #calculate R-factor - r1_factor_bin = ma_common_a.r1_factor(ma_common_b, use_binning=True) - r1_factor_bin.show() - r1_factor = ma_common_a.r1_factor(ma_common_b, use_binning=False) - print('Overall R-factor:', r1_factor) diff --git a/prime/command_line/ray_trace.py b/prime/command_line/ray_trace.py deleted file mode 100644 index dc300ab3df7..00000000000 --- a/prime/command_line/ray_trace.py +++ /dev/null @@ -1,157 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.ray_trace -""" -Author : Uervirojnangkoorn, M. -Created : 11/1/2015 -Description : read integration pickles and view systemetic absences and beam X, Y position -""" -from __future__ import absolute_import, division, print_function - -from six.moves import cPickle as pickle -from cctbx.array_family import flex -from cctbx.uctbx import unit_cell -from scitbx.matrix import sqr -import sys, math -import numpy as np -import matplotlib.pyplot as plt -from prime.postrefine.mod_leastsqr import good_unit_cell -from prime.postrefine.mod_partiality import partiality_handler -from prime.postrefine.mod_mx import mx_handler -from prime.postrefine.mod_leastsqr import good_unit_cell -from prime.postrefine.mod_input import read_frame, read_pickles -from six.moves import range -from six.moves import zip - -def read_input(args): - if len(args) == 0: - print("prime.ray_trace: read integration pickles and ray trace the reflections") - print("usage: prime.ray_trace data=integrated.lst") - exit() - data = [] - hklrefin = None - pixel_size_mm = None - target_unit_cell = None - d_min = 0 - d_max = 99 - for i in range(len(args)): - pair=args[i].split('=') - if pair[0]=='data': - data.append(pair[1]) - if pair[0]=='hklrefin': - hklrefin = pair[1] - if pair[0]=='pixel_size_mm': - pixel_size_mm = float(pair[1]) - if pair[0]=='target_unit_cell': - try: - tuc = pair[1].split(',') - a,b,c,alpha,beta,gamma = (float(tuc[0]), float(tuc[1]), float(tuc[2]), \ - float(tuc[3]), float(tuc[4]), float(tuc[5])) - target_unit_cell = unit_cell((a,b,c,alpha,beta,gamma)) - except Exception: - pass - if pair[0]=='d_min': - d_min = float(pair[1]) - if pair[0]=='d_max': - d_max = float(pair[1]) - if len(data)==0: - print("Please provide data path. (eg. data=/path/to/pickle/)") - exit() - if pixel_size_mm is None: - print("Please specify pixel size (eg. pixel_size_mm=0.079346)") - exit() - return data, hklrefin, pixel_size_mm, target_unit_cell, d_min, d_max - - -if (__name__ == "__main__"): - uc_tol = 3 - ry, rz, re, rotx, roty = (0, 0, 0.008, 0, 0) - flag_beam_divergence = False - lambda_template = flex.double(range(-50,50,1))/1000 - #0 .read input parameters and frames (pickle files) - data, hklrefin, pixel_size_mm, target_unit_cell, \ - d_min, d_max = read_input(args = sys.argv[1:]) - frame_files = read_pickles(data) - for pickle_filename in frame_files: - observations_pickle = read_frame(pickle_filename) - pickle_filename_arr = pickle_filename.split('/') - pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr)-1] - mxh = mx_handler() - flag_hklisoin_found, miller_array_iso = mxh.get_miller_array_from_reflection_file(hklrefin) - observations = observations_pickle["observations"][0] - #check if the uc is good - flag_good_unit_cell = good_unit_cell(observations.unit_cell().parameters(), None, uc_tol, target_unit_cell=target_unit_cell) - #update lambda_set - lambda_set = lambda_template + observations_pickle["wavelength"] - crystal_init_orientation = observations_pickle["current_orientation"][0] - detector_distance_mm = observations_pickle['distance'] - mm_predictions = pixel_size_mm*(observations_pickle['mapped_predictions'][0]) - xbeam = observations_pickle["xbeam"] - ybeam = observations_pickle["ybeam"] - alpha_angle = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ - for pred in mm_predictions]) - spot_pred_x_mm = flex.double([pred[0]-xbeam for pred in mm_predictions]) - spot_pred_y_mm = flex.double([pred[1]-ybeam for pred in mm_predictions]) - #resoultion filter - i_sel_res = observations.resolution_filter_selection(d_min=d_min, d_max=d_max) - observations = observations.select(i_sel_res) - alpha_angle = alpha_angle.select(i_sel_res) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res) - #collect offset spots (left and right) - off_left_x_mm = flex.double() - off_left_y_mm = flex.double() - off_right_x_mm = flex.double() - off_right_y_mm = flex.double() - off_mid_x_mm = flex.double() - off_mid_y_mm = flex.double() - if flag_good_unit_cell and len(spot_pred_x_mm) > 0: - #calculate rh set for lambda_set - rh_array = np.zeros([len(observations.data()), len(lambda_set)]) - p_array = np.zeros([len(observations.data()), len(lambda_set)]) - ph = partiality_handler() - r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), - observations.indices(), observations_pickle["wavelength"]) - for lambda_now, i_lambda in zip(lambda_set, range(len(lambda_set))): - two_theta = observations.two_theta(wavelength=lambda_now).data() - partiality, delta_xy, rs_set, rh_set = ph.calc_partiality_anisotropy_set(\ - crystal_init_orientation.unit_cell(), - rotx, roty, observations.indices(), - ry, rz, r0, re, 0, two_theta, alpha_angle, - lambda_now, crystal_init_orientation, - spot_pred_x_mm, spot_pred_y_mm, - detector_distance_mm, "Lorentzian", - flag_beam_divergence) - rh_array[:, i_lambda] = list(flex.abs(rh_set)) - p_array[:, i_lambda] = list(partiality) - #find minimum distance for all reflections - assigned_lambda = np.argmin(rh_array, axis=1) - #assigned_lambda = np.argmax(p_array, axis=1) - for i in range(len(observations.data())): - if assigned_lambda[i] == 0: - off_left_x_mm.append(spot_pred_x_mm[i]) - off_left_y_mm.append(spot_pred_y_mm[i]) - elif assigned_lambda[i] == len(lambda_template)-1: - off_right_x_mm.append(spot_pred_x_mm[i]) - off_right_y_mm.append(spot_pred_y_mm[i]) - else: - off_mid_x_mm.append(spot_pred_x_mm[i]) - off_mid_y_mm.append(spot_pred_y_mm[i]) - if assigned_lambda[i] == 0 or assigned_lambda[i] == len(lambda_template)-1: - dummy = 0 - # uncomment below to see best wavelength for each spot - """ - print p_array[i,:], assigned_lambda[i] - plt.plot(lambda_set, p_array[i,:]) - plt.title('Best wavelength=%8.6f (i=%2.0f) res=%6.2f'%(lambda_set[assigned_lambda[i]], assigned_lambda[i], observations.d_spacings().data()[i])) - plt.show() - """ - # plot which part of the spectrum this reflection is close to - plt.scatter(off_mid_x_mm, off_mid_y_mm, s=10, c='g', marker='o') - plt.scatter(off_left_x_mm, off_left_y_mm, s=20, c='b', marker='<') - plt.scatter(off_right_x_mm, off_right_y_mm, s=20, c='r', marker='>') - plt.title("Ray Trace Result o=right on, <=high energy, > low energy"); - plt.xlim([-100,100]) - plt.ylim([-100,100]) - plt.grid(True) - plt.show() - n, bins, patches = plt.hist(assigned_lambda, 30, normed=0, facecolor='b', alpha=0.5) - plt.show() diff --git a/prime/command_line/run.py b/prime/command_line/run.py deleted file mode 100644 index 7ad17f8d9b0..00000000000 --- a/prime/command_line/run.py +++ /dev/null @@ -1,21 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.run -""" handle prime run """ -from __future__ import absolute_import, division, print_function -__author__ = 'Monarin Uervirojnangkoorn, monarin@gmail.com' - -from subprocess import call -from prime.postrefine.mod_run import run_handler -from prime.postrefine.mod_input import process_input -import sys - -if __name__ == "__main__": - iparams, txt_out = process_input(sys.argv[1:] if len(sys.argv) > 1 else None, flag_mkdir=False) - if iparams.queue.mode: - args = ["bsub","-q",iparams.queue.qname,"-n", str(iparams.queue.n_nodes), "prime.postrefine"]+sys.argv[1:] if len(sys.argv) > 1 else [] - call(args) - print("Submitting prime job to ", iparams.queue.qname) - runh = run_handler() - runh.check_done(iparams) - else: - import prime.command_line.postrefine - prime.command_line.postrefine.run(sys.argv[1:]) diff --git a/prime/command_line/solve_indexing_ambiguity.py b/prime/command_line/solve_indexing_ambiguity.py deleted file mode 100644 index 35aa1cb523f..00000000000 --- a/prime/command_line/solve_indexing_ambiguity.py +++ /dev/null @@ -1,243 +0,0 @@ -''' -Author : Uervirojnangkoorn, M. -Created : 8/15/2016 -Description : Command line for solving indexing ambiguity -''' -from __future__ import absolute_import, division, print_function -import numpy as np -from libtbx.easy_mp import parallel_map -from prime.index_ambiguity.mod_indexing_ambiguity import indamb_handler -from six.moves import cPickle as pickle -from prime.index_ambiguity.mod_kmeans import kmeans_handler -from prime.postrefine.mod_mx import mx_handler -import random -from six.moves import range -from six.moves import zip - -def solve_with_mtz_mproc(args): - frame_no, pickle_filename, iparams, miller_array_ref = args - idah = indamb_handler() - index_basis, txt_out = idah.calc_cc(frame_no, pickle_filename, iparams, miller_array_ref) - print(txt_out) - return pickle_filename, index_basis - -def calculate_r_mproc(args): - frame_no, pickle_filename, index_basis, obs_main, obs_list = args - idah = indamb_handler() - r_set, txt_out = idah.calc_r(frame_no, pickle_filename, index_basis, obs_main, obs_list) - print(txt_out) - return pickle_filename, index_basis, r_set - -def get_obs_mproc(args): - frame_no, pickle_filename, iparams = args - idah = indamb_handler() - alt_dict = idah.get_observations(pickle_filename, iparams) - return alt_dict, pickle_filename - -class indexing_ambiguity_handler(object): - def __init__(self): - """ - Intialize parameters - """ - def get_twin_operators(self, obs): - idah = indamb_handler() - return idah.generate_twin_operators(obs_in) - - def should_terminate(self, iparams, pickle_filename): - #if no indexing ambiguity problem detected and mode is not set to "Forced" - idah = indamb_handler() - alt_dict = idah.get_observations(pickle_filename, iparams) - if len(alt_dict) == 1 and iparams.indexing_ambiguity.mode != 'Forced': - return True - if iparams.indexing_ambiguity.mode == 'Forced' and len(iparams.indexing_ambiguity.assigned_basis) == 0: - return True - return False - - def run(self, args): - import time - start = time.time() - #read inputs - from prime.postrefine.mod_input import process_input, read_pickles - iparams, txt_out_input = process_input(args) - print(txt_out_input) - f = open(iparams.run_no+'/log.txt', 'w') - f.write(txt_out_input) - f.close() - #if solution pickle is given, return the file name - if iparams.indexing_ambiguity.index_basis_in is not None: - if iparams.indexing_ambiguity.index_basis_in.endswith('.pickle'): - sol_pickle = pickle.load(open(iparams.indexing_ambiguity.index_basis_in, "rb")) - return sol_pickle, iparams - #read all integration pickles - frame_files = read_pickles(iparams.data) - n_frames = len(frame_files) - if n_frames == 0: - print("No integration pickle found. Exit program.") - return None, iparams - #exit if no problem - if self.should_terminate(iparams, frame_files[0]): - print("No indexing ambiguity problem. Set index_ambiguity.mode = Forced and assigned_basis = list of basis formats to solve pseudo-twinning problem.") - return None, iparams - #continue with (Auto - alt>1, find solution), (Auto - alt>1, mtz) - #(Forced - assigned_basis, mtz), (Forced - assigned_basis, find solution) - #************************************************* - #if mtz file is given, use it to solve the problem - sol_fname = iparams.run_no+'/index_ambiguity/solution_pickle.pickle' - if iparams.indexing_ambiguity.index_basis_in is not None: - if iparams.indexing_ambiguity.index_basis_in.endswith('.mtz'): - mxh = mx_handler() - flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(iparams.indexing_ambiguity.index_basis_in) - if flag_ref_found == False: - print("Reference mtz file not found. Set indexing_ambiguity.index_basis_in = None to enable auto generate the solutions.") - return None, iparams - else: - frames = [(i, frame_files[i], iparams, miller_array_ref) for i in range(n_frames)] - cc_results = parallel_map( - iterable=frames, - func=solve_with_mtz_mproc, - processes=iparams.n_processors) - sol_pickle = {} - for result in cc_results: - pickle_filename, index_basis = result - sol_pickle[pickle_filename] = index_basis - pickle.dump(sol_pickle, open(sol_fname,"wb")) - return sol_pickle, iparams - #************************************************* - #solve with Brehm & Diederichs - sample size n_sample_frames then bootstrap the rest - frames = [(i, frame_files[i], iparams) for i in random.sample(range(n_frames), iparams.indexing_ambiguity.n_sample_frames)] - #get observations list - print("Reading observations") - alt_dict_results = parallel_map( - iterable=frames, - func=get_obs_mproc, - processes=iparams.n_processors) - frame_dup_files = [] - frame_keys = [] - obs_list = [] - for result in alt_dict_results: - alt_dict, pickle_filename = result - if alt_dict is not None: - for key in alt_dict.keys(): - frame_dup_files.append(pickle_filename) - frame_keys.append(key) - obs_list.append(alt_dict[key]) - frames = [(i, frame_dup_files[i], frame_keys[i], obs_list[i], obs_list) for i in range(len(frame_dup_files))] - #calculate r - print("Calculating R") - calc_r_results = parallel_map( - iterable=frames, - func=calculate_r_mproc, - processes=iparams.n_processors) - frame_dup_files = [] - frame_keys = [] - r_matrix = [] - for result in calc_r_results: - if result is not None: - pickle_filename, index_basis, r_set = result - frame_dup_files.append(pickle_filename) - frame_keys.append(index_basis) - if len(r_matrix) == 0: - r_matrix = r_set - else: - r_matrix = np.append(r_matrix, r_set, axis=0) - #choose groups with best CC - print("Selecting frames with best R") - i_mean_r = np.argsort(np.mean(r_matrix, axis=1))[::-1] - r_matrix_sorted = r_matrix[i_mean_r] - frame_dup_files_sorted = np.array(frame_dup_files)[i_mean_r] - frame_keys_sorted = np.array(frame_keys)[i_mean_r] - frame_dup_files_sel = [] - for frame_file, frame_key, r_set in zip(frame_dup_files_sorted, frame_keys_sorted, r_matrix_sorted): - if frame_file not in frame_dup_files_sel: - frame_dup_files_sel.append(frame_file) - print(frame_file, frame_key, np.mean(r_set)) - if len(frame_dup_files_sel) >= iparams.indexing_ambiguity.n_selected_frames: - print('Found all %6.0f good frames'%(len(frame_dup_files_sel))) - break - ## - #rebuild observations and r_matrix - frames = [(i, frame_dup_files_sel[i], iparams) for i in range(len(frame_dup_files_sel))] - #get observations list - print("Re-reading observations") - alt_dict_results = parallel_map( - iterable=frames, - func=get_obs_mproc, - processes=iparams.n_processors) - frame_dup_files = [] - frame_keys = [] - obs_list = [] - for result in alt_dict_results: - alt_dict, pickle_filename = result - if alt_dict is not None: - for key in alt_dict.keys(): - frame_dup_files.append(pickle_filename) - frame_keys.append(key) - obs_list.append(alt_dict[key]) - frames = [(i, frame_dup_files[i], frame_keys[i], obs_list[i], obs_list) for i in range(len(frame_dup_files))] - #calculate r - print("Re-calculating R") - calc_r_results = parallel_map( - iterable=frames, - func=calculate_r_mproc, - processes=iparams.n_processors) - frame_dup_files = [] - frame_keys = [] - r_matrix = [] - for result in calc_r_results: - if result is not None: - pickle_filename, index_basis, r_set = result - frame_dup_files.append(pickle_filename) - frame_keys.append(index_basis) - if len(r_matrix) == 0: - r_matrix = r_set - else: - r_matrix = np.append(r_matrix, r_set, axis=0) - print("Minimizing frame distance") - idah = indamb_handler() - x_set = idah.optimize(r_matrix, flag_plot=iparams.flag_plot) - x_pickle = {'frame_dup_files':frame_dup_files, 'frame_keys':frame_keys, \ - 'r_matrix':r_matrix, 'x_set':x_set} - pickle.dump(x_pickle, open(iparams.run_no+'/index_ambiguity/x.out',"wb")) - print("Clustering results") - kmh = kmeans_handler() - k = 2**(len(idah.get_observations(frame_dup_files[0], iparams))-1) - centroids, labels = kmh.run(x_set, k, flag_plot=iparams.flag_plot) - print("Get solution pickle") - sample_fname = iparams.run_no+'/index_ambiguity/sample.lst' - sol_pickle = idah.assign_basis(frame_dup_files, frame_keys, labels, k, sample_fname) - pickle.dump(sol_pickle, open(sol_fname,"wb")) - #if more frames found, merge the sample frames to get a reference set - #that can be used for breaking the ambiguity. - txt_merge_out = None - if n_frames > iparams.indexing_ambiguity.n_selected_frames: - print("Breaking the indexing ambiguity for the remaining images.") - old_iparams_data = iparams.data[:] - iparams.indexing_ambiguity.index_basis_in = sol_pickle - #generate a reference set from solved frames - with open(sample_fname) as f: - frame_files_processed = f.read().split('\n')[:-1] - from prime.command_line.postrefine import scale_frames, merge_frames - scaled_pres_set = scale_frames(range(len(frame_files_processed)), frame_files_processed, iparams) - mdh, txt_merge_out = merge_frames(scaled_pres_set, iparams, mtz_out_prefix='index_ambiguity/ref') - miller_array_ref = mdh.miller_array_merge - #setup a list of remaining frames - frame_files_remain = [frame for frame in frame_files if frame not in sol_pickle] - frames = [(i, frame_files_remain[i], iparams, miller_array_ref) for i in range(len(frame_files_remain))] - cc_results = parallel_map( - iterable=frames, - func=solve_with_mtz_mproc, - processes=iparams.n_processors) - for result in cc_results: - pickle_filename, index_basis = result - sol_pickle[pickle_filename] = index_basis - iparams.data = old_iparams_data[:] - #write out solution pickle - pickle.dump(sol_pickle, open(sol_fname,"wb")) - #write out text output - txt_out = "Solving indexing ambiguity complete. Solution file saved to "+sol_fname+"\n" - if txt_merge_out: - txt_out += "Reference set used to solve the indexing ambiguity problem:\n"+txt_merge_out - with open(iparams.run_no+'/log.txt', 'a') as f: - f.write(txt_out) - print("Indexing Ambiguity Solver Elapsed Time (s) %10.2s"%(time.time()-start)) - return sol_pickle, iparams diff --git a/prime/command_line/spot_param_visualizer.py b/prime/command_line/spot_param_visualizer.py deleted file mode 100644 index 1868757019a..00000000000 --- a/prime/command_line/spot_param_visualizer.py +++ /dev/null @@ -1,37 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME iota.double_gauss_visualizer -from __future__ import absolute_import, division, print_function -import logging -import prime.iota.double_gauss as dg - -FORMAT = '%(levelname)s %(module)s.%(funcName)s: %(message)s' -logging.basicConfig(level=logging.INFO, format=FORMAT) - -def run(args): - if args < 2: - raise IOError("Must specify an integration pickle image") - img_data = dg.get_img_data(args.image) - spot_man = dg.SpotManager(img_data, args.t, args.b, args.s, - args.threshold_range, args.background_range, - args.smooth_range) - spot_man.plot_spots() - -if __name__ == "__main__": - - import argparse - parser = argparse.ArgumentParser(description=('Visually optimize double-Gauss spotfinder')) - parser.add_argument('image', type=str, - help='Path to an image pickle.') - parser.add_argument('-t', type=float, default=1.5, - help='Initial threshold value for spotfinding.') - parser.add_argument('-b', type=float, default=10, - help='Initial threshold value for background Gaussian kernel.') - parser.add_argument('-s', type=float, default=1.5, - help='Initial threshold value for spot Gaussian kernel.') - parser.add_argument('--threshold-range', type=float, nargs=2, default=(1, 4), - help="Range for threshold") - parser.add_argument('--background-range', type=float, nargs=2, default=(0,100), - help="Range for background slider") - parser.add_argument('--smooth-range', type=float, nargs=2, default=(0, 15), - help="Range for smoothing slider") - args = parser.parse_args() - run(args) diff --git a/prime/command_line/view_fineslices.py b/prime/command_line/view_fineslices.py deleted file mode 100644 index 6d043e90ccf..00000000000 --- a/prime/command_line/view_fineslices.py +++ /dev/null @@ -1,80 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.view_fineslices -""" -Author : Uervirojnangkoorn, M. -Desc : Read pickle files (hopefully fine-sliced, and named after the - order of the slices. Grab selected reflections and plot the - intensity. -""" -from __future__ import absolute_import, division, print_function -import sys -from six.moves import cPickle as pickle -from dials.array_family import flex -import matplotlib.pyplot as plt -import numpy as np -from prime.postrefine.mod_input import read_frame, read_pickles -from six.moves import range -from six.moves import zip - -def read_input(args): - data = [] - data_sweep = '' - for i in range(len(args)): - pair=args[i].split('=') - if len(pair) == 2: - if pair[0]=='data': - data.append(pair[1]) - elif pair[0]=='data_sweep': - data_sweep = pair[1] - if len(data) == 0: - print("Please input all parameters") - exit() - return data, data_sweep - -if (__name__ == "__main__"): - #Read input parameters and frames (pickle files) - data_fine, data_sweep = read_input(args = sys.argv[1:]) - pixel_size_mm = 0.079346 - #read all the pickle files from fine-slice data - frames_fine = read_pickles(data_fine) - #get a sample reflections - sample_no = 0 - obs_fine_sample = None - for i in range(2000): - frame = frames_fine[i] - pickle_fine = read_frame(frame) - obs_fine = pickle_fine["observations"][0] - obs_fine = obs_fine.select(obs_fine.data() > 0) - if len(obs_fine.data())> 5: - print(frame) - for index, d, I, sigI in zip(obs_fine.indices(), obs_fine.d_spacings().data(),\ - obs_fine.data(), obs_fine.sigmas()): - print(index, d, I, sigI) - obs_fine_sample = obs_fine.deep_copy() - #break - sample_no += 1 - """ - #find matching indices in the next consecutive frames - n_frames_sel = 10 - colors = np.random.rand(n_frames_sel) - i = 0 - for frame in frames_fine[sample_no:sample_no+n_frames_sel]: - pickle_fine = read_frame(frame) - obs_fine = pickle_fine["observations"][0] - mm_predictions = pixel_size_mm * (pickle_fine['mapped_predictions'][0]) - xbeam = pickle_fine["xbeam"] - ybeam = pickle_fine["ybeam"] - spot_pred_x_mm = flex.double([pred[0]-xbeam for pred in mm_predictions]) - spot_pred_y_mm = flex.double([pred[1]-ybeam for pred in mm_predictions]) - #plt.subplot(2,5,i+1) - plt.scatter(spot_pred_x_mm, spot_pred_y_mm, s=10, c=colors) - #plt.title(frame) - plt.grid(True) - obs_fine = obs_fine.select(obs_fine.data() > 0) - obs_fine_sample_match, obs_fine_match = obs_fine_sample.common_sets(obs_fine) - print frame - for index, d, I, sigI in zip(obs_fine_match.indices(), obs_fine_match.d_spacings().data(),\ - obs_fine_match.data(), obs_fine_match.sigmas()): - print index, d, I, sigI - i += 1 - plt.show() - """ diff --git a/prime/command_line/viewstats.py b/prime/command_line/viewstats.py deleted file mode 100644 index b0b72df3fa6..00000000000 --- a/prime/command_line/viewstats.py +++ /dev/null @@ -1,120 +0,0 @@ -# LIBTBX_SET_DISPATCHER_NAME prime.viewstats -''' -Author : Uervirojnangkoorn, M. -Created : 9/19/2014 -Description : View convergence and other stats for post-refinement. -''' -from __future__ import absolute_import, division, print_function -import matplotlib.pyplot as plt -import sys -import numpy as np -from six.moves import range - -if len(sys.argv)==1: - print('Use prime.viewstats to view convergence of post-refined parameters.') - print('Usage: prime.viewstats your_run_no') - exit() -run_no = sys.argv[1] - -#read .paramhist files and display refinement results -import os -cn_file = 0 -for file_in in os.listdir(run_no): - if file_in.endswith('.paramhist'): - cn_file += 1 - -if cn_file == 0: - print('Cannot find .paramhist file in your ', run_no, '.') - print('To enable viewstats, rerun prime with flag_output_verbose=True in .phil file.') - print('The .paramhist parameters will be recorded during the run.') - exit() - -param_file_list = [] -for i in range(cn_file): - param_file_list.append(run_no+'/'+str(i)+'.paramhist') - -data_dict_list = [] -for param_file in param_file_list: - pf = open(param_file,'r') - data = pf.read().split('\n') - data_dict = {} - n_data = 0 - for data_row in data: - dc = data_row.split() - #use row 1 to set n_col - if n_data == 0: - n_col = len(dc) - if len(dc)==n_col: - data_dict[dc[n_col-1]] = np.array([float(dc[i]) for i in range(n_col-1)]) - n_data += 1 - data_dict_list.append(data_dict) - -#prepare test key -data_dict_0 = data_dict_list[0] -data_dict_1 = data_dict_list[1] -for i in range(n_data): - test_key = list(data_dict_list[0].keys())[i] - if (test_key in data_dict_0) and (test_key in data_dict_1): - test_id = i - break - -#Fix Tpr and Txy for first data_dict -test_param_0_raw = np.array(data_dict_0[test_key]) -test_param_1 = data_dict_1[test_key] -for key in data_dict_0.keys(): - if key in data_dict_1: - data_param_0 = data_dict_0[key] - data_param_1 = data_dict_1[key] - data_param_0[1] = data_param_1[0] - data_param_0[3] = data_param_1[2] - data_dict_0[key] = data_param_0 - -test_param_0_update = data_dict_0[test_key] -test_delta_1_calc = np.absolute(test_param_1-test_param_0_update) -print('test id', test_id) -print('test key', test_key) -print('0th cycle (raw):', test_param_0_raw) -print('0th cycle (updated):', test_param_0_update) -print('1st cycle:', test_param_1) -print('delta (calc.):', test_delta_1_calc) - -delta_dict_list = [] -for i in range(len(data_dict_list)-1): - data_dict = data_dict_list[i] - data_dict_next = data_dict_list[i+1] - delta_dict = {} - for key in data_dict.keys(): - if (key in data_dict_next): - delta_param = np.absolute(data_dict_next[key] - data_dict[key]) - else: - delta_param = np.zeros(n_col-1) - delta_dict[key] = delta_param - delta_dict_list.append(delta_dict) - -delta_dict_0 = delta_dict_list[0] -test_delta_1 = delta_dict_0[test_key] -print('delta (prog.):', test_delta_1) -print('delta diff.:', test_delta_1 - test_delta_1_calc) -print('sum of delta diff.', np.sum(np.absolute(test_delta_1 - test_delta_1_calc))) - -x_range = list(range(1, len(delta_dict_list)+1)) -x_label = [] -for i in range(1, len(delta_dict_list)+1): - x_label.append(str(i)) -data_title = ['Tpr_i','Tpr','Txy_i','Txy','G','B','RotX','RotY','ry','rz','r0','re','voigt_nu','a','b','c','alpha','beta','gamma','CC1/2'] -cn_plot = 1 -for i in range(n_col-1): - if i not in (0,2): - data_series = [] - for delta_dict in delta_dict_list: - narr = np.array([delta_dict[key][i] for key in delta_dict.keys()]) - data_series.append(narr) - - ax = plt.subplot(3, 6, cn_plot, title=data_title[i]) - plt.boxplot(data_series) - plt.xticks(x_range, x_label) - if data_title[i] in ('ry','rz','r0','re'): - plt.ylim([0, 0.01]) - plt.grid(True) - cn_plot += 1 -plt.show() diff --git a/prime/ext.cpp b/prime/ext.cpp deleted file mode 100644 index 4cbe8ac8c6e..00000000000 --- a/prime/ext.cpp +++ /dev/null @@ -1,604 +0,0 @@ -/* -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil; tab-width: 8 -*- - * - * $Id: ext.cpp 21225 2014-11-28 21:06:02Z phyy-nx $ - */ -#include -#include -#include -#include -#include -#include -#include -#include - -/* - -This file contains an implmentation of calc_avg_I from -postrefine/mod_util.py, which averages observed intensities, -does outlier rejection and computes various statistics. - -*/ - -using namespace boost::python; - -namespace prime { - enum Average_Mode { - Average, ///< normal avearaging - Weighted, ///< weighted averaging - Final ///< averaging done when the postrefinement is finishing - }; - - typedef - scitbx::af::shared > shared_miller; - - struct average_result_store { - // class for returning results - shared_miller miller_index; - scitbx::af::shared I_avg; - scitbx::af::shared sigI_avg; - scitbx::af::shared r_meas_w_top; - scitbx::af::shared r_meas_w_btm; - scitbx::af::shared r_meas_top; - scitbx::af::shared r_meas_btm; - scitbx::af::shared multiplicity; - scitbx::af::shared I_avg_even; - scitbx::af::shared I_avg_odd; - scitbx::af::shared I_avg_even_h; - scitbx::af::shared I_avg_odd_h; - scitbx::af::shared I_avg_even_k; - scitbx::af::shared I_avg_odd_k; - scitbx::af::shared I_avg_even_l; - scitbx::af::shared I_avg_odd_l; - std::string txt_obs_out; - std::string txt_reject_out; - }; - - class averaging_engine { - // interface for computing averages - const int group_no_; - const scitbx::af::shared group_id_list_; - const shared_miller miller_index_; - const shared_miller miller_index_ori_; - const scitbx::af::shared I_; - const scitbx::af::shared sigI_; - const scitbx::af::shared G_; - const scitbx::af::shared B_; - const scitbx::af::shared p_set_; - const scitbx::af::shared rs_set_; - const scitbx::af::shared wavelength_set_; - const scitbx::af::shared sin_theta_over_lambda_sq_; - const scitbx::af::shared SE_; - const scitbx::af::shared pickle_filename_set_; - public: - Average_Mode avg_mode_; - double sigma_max_; - bool flag_volume_correction_; - int n_rejection_cycle_; - bool flag_output_verbose_; - - public: averaging_engine( - // main constructor, used for passing in arrays. other parmaters exposed - // as properties to python - int group_no, - scitbx::af::shared group_id_list, - const shared_miller& miller_index, - const shared_miller& miller_index_ori, - const scitbx::af::shared& I, - const scitbx::af::shared& sigI, - const scitbx::af::shared& G, - const scitbx::af::shared& B, - const scitbx::af::shared& p_set, - const scitbx::af::shared& rs_set, - const scitbx::af::shared& wavelength_set, - const scitbx::af::shared& sin_theta_over_lambda_sq, - const scitbx::af::shared& SE, - const scitbx::af::shared& pickle_filename_set - ): - group_no_(group_no), - group_id_list_(group_id_list), - miller_index_(miller_index), - miller_index_ori_(miller_index_ori), - I_(I),sigI_(sigI),G_(G),B_(B),p_set_(p_set), - rs_set_(rs_set), - wavelength_set_(wavelength_set), - sin_theta_over_lambda_sq_(sin_theta_over_lambda_sq), - SE_(SE), - pickle_filename_set_(pickle_filename_set) - { - avg_mode_ = Average; - sigma_max_ = 99.0; - flag_volume_correction_ = true; - n_rejection_cycle_ = 1; - flag_output_verbose_ = false; - } - - void calc_avg_two_halves( - const scitbx::af::shared& I_full_group, - const scitbx::af::shared& SE_norm, - const Average_Mode& avg_mode_, - double& I_avg_even, - double& I_avg_odd - ) - { - double I_even_sum = 0; - double I_odd_sum = 0; - double I_even_weighted_sum = 0; - double I_odd_weighted_sum = 0; - double SE_norm_even_sum = 0; - double SE_norm_odd_sum = 0; - I_avg_even = 0; - I_avg_odd = 0; - if (I_full_group.size() > 2) { - for (int i = 0; i < I_full_group.size(); i++) { - if (i % 2 == 0) { - I_even_sum += I_full_group[i]; - I_even_weighted_sum += I_full_group[i] * SE_norm[i]; - SE_norm_even_sum += SE_norm[i]; - } - else { - I_odd_sum += I_full_group[i]; - I_odd_weighted_sum += I_full_group[i] * SE_norm[i]; - SE_norm_odd_sum += SE_norm[i]; - } - } - - if (I_full_group.size() % 2 == 1) { - I_odd_sum += I_full_group[I_full_group.size()-1]; - I_odd_weighted_sum += I_full_group[I_full_group.size()-1] * SE_norm[SE_norm.size()-1]; - SE_norm_odd_sum += SE_norm[SE_norm.size()-1]; - } - - if (avg_mode_ == Weighted || avg_mode_ == Final) { - I_avg_even = I_even_weighted_sum/SE_norm_even_sum; - I_avg_odd = I_odd_weighted_sum/SE_norm_odd_sum; - } - else { - SCITBX_ASSERT(avg_mode_ == Average); - int size; - if(I_full_group.size() % 2 == 1) - size = (I_full_group.size()+1)/2; - else - size = I_full_group.size()/2; - I_avg_even = I_even_sum/size; - I_avg_odd = I_odd_sum/size; - } - } - } - - static const double CONST_SE_MIN_WEIGHT; - static const double CONST_SE_MAX_WEIGHT; - static const double CONST_SIG_I_FACTOR; - - public: average_result_store - calc_avg_I() { - /* - Average the intensites. - - I_ is an array with N observations. miller_indices is an array with M indices. - M should always be <= N. Observations are arranged into groups where the members - of the same group have the same miller index. group_id_list is N long and indexes - each observation to a group and its miller index. group_id_list should be sorted - (ascending) and that sort order applied to G, B, etc. - */ - - average_result_store results; - std::ostringstream txt_obs_out; - std::ostringstream txt_reject_out; - - // convert to the full intensity and calculate mosaic spread (currently only logged) - scitbx::af::shared I_full; - scitbx::af::shared sigI_full; - scitbx::af::shared mosaic_radian_set; - for(int x = 0; x < G_.size(); x++) { - double tmp1 = G_[x] *std::exp(-2*B_[x]*sin_theta_over_lambda_sq_[x]); - double tmp2 = I_[x]; - double tmp3 = p_set_[x]; - if (x==0) - printf("c++ check overflow: %70.70f\n",tmp1*tmp2); // Without this printf, I_full comes out differently between - // python and cpp in the highest significant digits. No idea why. - I_full.push_back(tmp2/(tmp1*tmp3)); - sigI_full.push_back(sigI_[x]/(G_[x] * std::exp(-2*B_[x]*sin_theta_over_lambda_sq_[x]) * p_set_[x])); - mosaic_radian_set.push_back(2 * rs_set_[x] * wavelength_set_[x]); - } - - if (flag_volume_correction_) - for (int x = 0; x < I_full.size(); x++){ - I_full[x] *= (4.0/3.0) * (rs_set_[x]); - sigI_full[x] *= (4.0/3.0) * (rs_set_[x]); - } - - // Iterate over each group of intensites. They will match a single miller_index each - int obs_ptr = 0; // this will track along the intensites array as each group is processed - for (int g = 0; g < group_no_; g++) { - SCITBX_ASSERT(group_id_list_[obs_ptr] == g); // this verifies that group_id_list is sorted - - int obs_ptr_start = obs_ptr; - - // gather intensites and errors for this group - double max_w = CONST_SE_MAX_WEIGHT; - double min_w = std::sqrt(CONST_SE_MIN_WEIGHT); - scitbx::af::shared I_group; - scitbx::af::shared sigI_group; - scitbx::af::shared I_full_group; - scitbx::af::shared sigI_full_group; - scitbx::af::shared SE_group; - cctbx::miller::index current_index; - shared_miller current_index_ori; - scitbx::af::sharedpickle_filename_set_group; - std::ostringstream txt_reject_out_group; - - scitbx::af::shared valid_ptrs; - - bool found_one = false; - while (obs_ptr < I_.size()) { - if (group_id_list_[obs_ptr] != g) { - break; // found them all - } - - if (found_one) { - SCITBX_ASSERT(current_index == miller_index_[obs_ptr]); - } - else { - found_one = true; - current_index = miller_index_[obs_ptr]; - } - I_group.push_back(I_[obs_ptr]); - sigI_group.push_back(sigI_[obs_ptr]); - I_full_group.push_back(I_full[obs_ptr]); - sigI_full_group.push_back(sigI_full[obs_ptr]); - SE_group.push_back(SE_[obs_ptr]); - current_index_ori.push_back(miller_index_ori_[obs_ptr]); - pickle_filename_set_group.push_back(pickle_filename_set_[obs_ptr]); - valid_ptrs.push_back(obs_ptr); - obs_ptr++; - } - SCITBX_ASSERT(found_one); - - // log - char buf[512]; - snprintf(buf, sizeof(buf), "Reflection: %d,%d,%d\nmeanI medI sigI_est sigI_true delta_sigI n_refl\n",current_index[0],current_index[1],current_index[2]); - txt_obs_out << buf; - scitbx::af::shared I_full_group_copy; - for (int i = 0; i < I_full_group.size(); i++) - I_full_group_copy.push_back(I_full_group[i]); - scitbx::math::basic_statistics basic_stat(I_full_group_copy.const_ref()); - scitbx::math::median_functor mf; - - double median_I = mf(I_full_group_copy.ref()); - double mean_I = basic_stat.mean; - double std_I = 0; - //if (I_full_group.size() > 1) - std_I = basic_stat.biased_standard_deviation; // based on what I think numpy is doing compared to basic_statistics - - snprintf(buf, sizeof(buf), "%6.2f %6.2f %8.2f %8.0f\n", mean_I, median_I, std_I, double(I_full_group.size())); - txt_obs_out << buf; - - //reject outliers - if (I_full_group.size() > 2) { - for (int i_rejection = 0; i_rejection < n_rejection_cycle_; i_rejection++) { - scitbx::af::shared I_full_group_copy; - for (int i = 0; i < I_full_group.size(); i++) - I_full_group_copy.push_back(I_full_group[i]); - scitbx::math::basic_statistics basic_stat(I_full_group_copy.const_ref()); - scitbx::math::median_functor mf; - - scitbx::af::shared I_group_filtered; - scitbx::af::shared sigI_group_filtered; - scitbx::af::shared I_full_group_filtered; - scitbx::af::shared sigI_full_group_filtered; - scitbx::af::shared SE_group_filtered; - shared_miller current_index_ori_filtered; - scitbx::af::shared pickle_filename_set_group_filtered; - - scitbx::af::shared valid_ptrs_filtered; - - double median_I = mf(I_full_group_copy.ref()); - double mean_I = basic_stat.mean; - double std_I = basic_stat.biased_standard_deviation; // based on what I think numpy is doing compared to basic_statistics - - for (int i = 0; i < I_full_group.size(); i++) { - double I_full_as_sigma = (I_full_group[i] - median_I) / std_I; - if (std::abs(I_full_as_sigma) > sigma_max_) { - char buf[512]; - snprintf(buf, sizeof(buf), "%s %3.0f %3.0f %3.0f %10.2f %10.2f\n", pickle_filename_set_group[i].c_str(), - double(current_index_ori[i][0]), double(current_index_ori[i][1]), double(current_index_ori[i][2]), I_group[i], sigI_group[i]); - txt_reject_out_group << buf; - } - else { - I_group_filtered.push_back(I_group[i]); - sigI_group_filtered.push_back(sigI_group[i]); - I_full_group_filtered.push_back(I_full_group[i]); - sigI_full_group_filtered.push_back(sigI_full_group[i]); - SE_group_filtered.push_back(SE_group[i]); - current_index_ori_filtered.push_back(current_index_ori[i]); - pickle_filename_set_group_filtered.push_back(pickle_filename_set_group[i]); - valid_ptrs_filtered.push_back(valid_ptrs[i]); - } - } - I_group = I_group_filtered; - sigI_group = sigI_group_filtered; - I_full_group = I_full_group_filtered; - sigI_full_group = sigI_full_group_filtered; - SE_group = SE_group_filtered; - current_index_ori = current_index_ori_filtered; - pickle_filename_set_group = pickle_filename_set_group_filtered; - valid_ptrs = valid_ptrs_filtered; - - char buf[512]; - snprintf(buf, sizeof(buf), "%6.2f %6.2f %8.2f %8.0f\n", mean_I, median_I, std_I, double(I_full_group.size())); - txt_obs_out << buf; - - if (I_full_group.size() <= 3) - break; - } - if (I_full_group.size() == 0) { - printf("miller_index (%d, %d, %d) rejected at calc_avg", current_index[0], current_index[1], current_index[2]); - continue; - } - } - // normalize the SE - scitbx::af::shared SE_norm; - double se_max = scitbx::af::max(SE_group.ref()); - double se_min = scitbx::af::min(SE_group.ref()); - double SE_norm_val; - if (SE_group.size() == 1 || ((se_max-se_min) < 0.1) || avg_mode_ == Average) { - SE_norm_val = 1; - for (int i = 0; i < SE_group.size(); i++) { - SE_norm.push_back(SE_norm_val); - } - } - else { - double m = (max_w - min_w)/(se_min-se_max); - double b = max_w - (m*se_min); - - for (int i = 0; i < SE_group.size(); i++) { - SE_norm.push_back( (m*SE_group[i]) + b); - } - } - - double SE_norm_sum = scitbx::af::sum(SE_norm.const_ref()); - SCITBX_ASSERT(SE_norm_sum != 0); - scitbx::af::shared avg_tmp; - for (int i = 0; i < SE_norm.size(); i++) { - avg_tmp.push_back(SE_norm[i] * I_full_group[i]); - } - - double I_avg = scitbx::af::sum(avg_tmp.const_ref())/SE_norm_sum; - double sigI_avg = scitbx::af::mean(sigI_full_group.const_ref()); - - //Rmeas, Rmeas_w, multiplicity - int multiplicity = I_full_group.size(); - double r_meas_w_top = 0; - double r_meas_w_btm = 0; - double r_meas_top = 0; - double r_meas_btm = 0; - double r_meas = 0; - double r_meas_w = 0; - if (multiplicity > 1) { - for (int i = 0; i < multiplicity; i++) { - r_meas_w_top += std::pow(((I_full_group[i] - I_avg)*SE_norm[i]),2); - r_meas_w_btm += std::pow(I_full_group[i]*SE_norm[i],2); - r_meas_top += std::abs(((I_full_group[i] - I_avg)*SE_norm[i])); - r_meas_btm += std::abs(I_full_group[i]*SE_norm[i]); - } - r_meas_w = r_meas_w_top/r_meas_w_btm; - r_meas = r_meas_top/r_meas_btm; - } - - //for calculation of cc1/2 - //separate the observations into two groups - double I_avg_even = 0; - double I_avg_odd = 0; - double I_avg_even_h = 0; - double I_avg_odd_h = 0; - double I_avg_even_k = 0; - double I_avg_odd_k = 0; - double I_avg_even_l = 0; - double I_avg_odd_l = 0; - - calc_avg_two_halves(I_full_group, SE_norm, avg_mode_, I_avg_even, I_avg_odd); - - //select reflections on h axis - scitbx::af::shared I_full_group_h; - scitbx::af::shared SE_norm_h; - scitbx::af::shared I_full_group_k; - scitbx::af::shared SE_norm_k; - scitbx::af::shared I_full_group_l; - scitbx::af::shared SE_norm_l; - for (int i = 0; i < I_full_group.size(); i++) { - if (current_index_ori[i][0] == 0) { - I_full_group_h.push_back(I_full_group[i]); - SE_norm_h.push_back(SE_norm[i]); - } - - if (current_index_ori[i][1] == 0) { - I_full_group_k.push_back(I_full_group[i]); - SE_norm_k.push_back(SE_norm[i]); - } - - if (current_index_ori[i][2] == 0) { - I_full_group_l.push_back(I_full_group[i]); - SE_norm_l.push_back(SE_norm[i]); - } - } - - - calc_avg_two_halves(I_full_group_h, SE_norm_h, avg_mode_, I_avg_even_h, I_avg_odd_h); - calc_avg_two_halves(I_full_group_k, SE_norm_k, avg_mode_, I_avg_even_k, I_avg_odd_k); - calc_avg_two_halves(I_full_group_l, SE_norm_l, avg_mode_, I_avg_even_l, I_avg_odd_l); - - - // save the results for this group - results.miller_index.push_back(current_index); - results.I_avg.push_back(I_avg); - results.sigI_avg.push_back(sigI_avg); - results.r_meas_w_top.push_back(r_meas_w_top); - results.r_meas_w_btm.push_back(r_meas_w_btm); - results.r_meas_top.push_back(r_meas_top); - results.r_meas_btm.push_back(r_meas_btm); - results.multiplicity.push_back(multiplicity); - results.I_avg_even.push_back(I_avg_even); - results.I_avg_odd.push_back(I_avg_odd); - results.I_avg_even_h.push_back(I_avg_even_h); - results.I_avg_odd_h.push_back(I_avg_odd_h); - results.I_avg_even_k.push_back(I_avg_even_k); - results.I_avg_odd_k.push_back(I_avg_odd_k); - results.I_avg_even_l.push_back(I_avg_even_l); - results.I_avg_odd_l.push_back(I_avg_odd_l); - - if (flag_output_verbose_) { - txt_obs_out << " I_o sigI_o G B Eoc rs lambda rocking(deg) W I_full sigI_full\n"; - for (int i = 0; i < I_full_group.size(); i++) { - char buf[512]; - snprintf(buf, sizeof(buf), "%10.2f %10.2f %6.2f %6.2f %6.2f %8.5f %8.5f %8.5f %6.2f %10.2f %10.2f\n", - I_group[i],sigI_group[i],1/G_[valid_ptrs[i]],B_[valid_ptrs[i]],p_set_[valid_ptrs[i]],rs_set_[valid_ptrs[i]], - wavelength_set_[valid_ptrs[i]],mosaic_radian_set[valid_ptrs[i]]*180/scitbx::constants::pi,SE_norm[i], - I_full_group[i],sigI_full[i]); - txt_obs_out << buf; - } - char buf[512]; - snprintf(buf, sizeof(buf), "Merged I, sigI: %6.2f, %6.2f\n",I_avg,sigI_avg); - txt_obs_out << buf; - snprintf(buf, sizeof(buf), "Rmeas: %6.2f Qw: %6.2f\n",r_meas,r_meas_w); - txt_obs_out << buf; - snprintf(buf, sizeof(buf), "No. total observed: %4.0f No. after rejection: %4.0f\n", double(obs_ptr-obs_ptr_start), double(I_full_group.size())); - txt_obs_out << buf; - txt_obs_out << "List of rejected observations:\n"; - txt_obs_out << txt_reject_out_group.str(); - } - txt_reject_out << txt_reject_out_group.str(); - } - results.txt_obs_out = txt_obs_out.str(); - results.txt_reject_out = txt_reject_out.str(); - - return results; - } - }; - - -const double averaging_engine::CONST_SE_MIN_WEIGHT = 0.17; -const double averaging_engine::CONST_SE_MAX_WEIGHT = 1.0; -const double averaging_engine::CONST_SIG_I_FACTOR = 1.5; - -namespace boost_python { namespace { - void - init_module() { - using namespace boost::python; - typedef return_value_policy rbv; - typedef default_call_policies dcp; - typedef averaging_engine w_t; - - class_("averaging_engine", no_init) - .def(init, - const shared_miller&,const shared_miller&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared&, - const scitbx::af::shared& - >( - (arg("group_no"),arg("group_id_list"), - arg("miller_list"),arg("miller_list_ori"),arg("I"),arg("sigI"), - arg("G"),arg("B"),arg("p_set"),arg("rs_set"), - arg("wavelength_set"),arg("sin_theta_over_lambda_sq"),arg("SE"), - arg("pickle_filename_set") - ))) - .def("calc_avg_I", &averaging_engine::calc_avg_I) - .add_property("avg_mode", - make_getter(&averaging_engine::avg_mode_, rbv()), - make_setter(&averaging_engine::avg_mode_, dcp())) - .add_property("sigma_max", - make_getter(&averaging_engine::sigma_max_, rbv()), - make_setter(&averaging_engine::sigma_max_, dcp())) - .add_property("flag_volume_correction", - make_getter(&averaging_engine::flag_volume_correction_, rbv()), - make_setter(&averaging_engine::flag_volume_correction_, dcp())) - .add_property("n_rejection_cycle", - make_getter(&averaging_engine::n_rejection_cycle_, rbv()), - make_setter(&averaging_engine::n_rejection_cycle_, dcp())) - .add_property("flag_output_verbose", - make_getter(&averaging_engine::flag_output_verbose_, rbv()), - make_setter(&averaging_engine::flag_output_verbose_, dcp())) - ; - - class_("average_result_store",init<>()) - .add_property("miller_index", - make_getter(&average_result_store::miller_index, rbv()), - make_setter(&average_result_store::miller_index, dcp())) - .add_property("I_avg", - make_getter(&average_result_store::I_avg, rbv()), - make_setter(&average_result_store::I_avg, dcp())) - .add_property("sigI_avg", - make_getter(&average_result_store::sigI_avg, rbv()), - make_setter(&average_result_store::sigI_avg, dcp())) - .add_property("r_meas_w_top", - make_getter(&average_result_store::r_meas_w_top, rbv()), - make_setter(&average_result_store::r_meas_w_top, dcp())) - .add_property("r_meas_w_btm", - make_getter(&average_result_store::r_meas_w_btm, rbv()), - make_setter(&average_result_store::r_meas_w_btm, dcp())) - .add_property("r_meas_top", - make_getter(&average_result_store::r_meas_top, rbv()), - make_setter(&average_result_store::r_meas_top, dcp())) - .add_property("r_meas_btm", - make_getter(&average_result_store::r_meas_btm, rbv()), - make_setter(&average_result_store::r_meas_btm, dcp())) - .add_property("multiplicity", - make_getter(&average_result_store::multiplicity, rbv()), - make_setter(&average_result_store::multiplicity, dcp())) - .add_property("I_avg_even", - make_getter(&average_result_store::I_avg_even, rbv()), - make_setter(&average_result_store::I_avg_even, dcp())) - .add_property("I_avg_odd", - make_getter(&average_result_store::I_avg_odd, rbv()), - make_setter(&average_result_store::I_avg_odd, dcp())) - .add_property("I_avg_even_h", - make_getter(&average_result_store::I_avg_even_h, rbv()), - make_setter(&average_result_store::I_avg_even_h, dcp())) - .add_property("I_avg_odd_h", - make_getter(&average_result_store::I_avg_odd_h, rbv()), - make_setter(&average_result_store::I_avg_odd_h, dcp())) - .add_property("I_avg_even_k", - make_getter(&average_result_store::I_avg_even_k, rbv()), - make_setter(&average_result_store::I_avg_even_k, dcp())) - .add_property("I_avg_odd_k", - make_getter(&average_result_store::I_avg_odd_k, rbv()), - make_setter(&average_result_store::I_avg_odd_k, dcp())) - .add_property("I_avg_even_l", - make_getter(&average_result_store::I_avg_even_l, rbv()), - make_setter(&average_result_store::I_avg_even_l, dcp())) - .add_property("I_avg_odd_l", - make_getter(&average_result_store::I_avg_odd_l, rbv()), - make_setter(&average_result_store::I_avg_odd_l, dcp())) - .add_property("txt_obs_out", - make_getter(&average_result_store::txt_obs_out, rbv()), - make_setter(&average_result_store::txt_obs_out, dcp())) - .add_property("txt_reject_out", - make_getter(&average_result_store::txt_reject_out, rbv()), - make_setter(&average_result_store::txt_reject_out, dcp())) - ; - }; - - using namespace boost::python; - - void export_average_mode() - { - enum_("Average_Mode") - .value("Average", Average) - .value("Weighted", Weighted) - .value("Final", Final); - } - -}}} // namespace prime::boost_python:: - -BOOST_PYTHON_MODULE(prime_ext) -{ - prime::boost_python::init_module(); - prime::boost_python::export_average_mode(); - -} diff --git a/prime/index_ambiguity/SConscript b/prime/index_ambiguity/SConscript deleted file mode 100644 index 4479bf8fdd6..00000000000 --- a/prime/index_ambiguity/SConscript +++ /dev/null @@ -1,23 +0,0 @@ -import libtbx.load_env -import os -Import("env_base", "env_etc") - -env = env_base.Clone(SHLINKFLAGS=env_etc.shlinkflags) -env.Append(LIBS=["cctbx"] + env_etc.libm) -env_etc.include_registry.append( - env=env, - paths=env_etc.prime_common_includes) -if (env_etc.static_libraries): builder = env.StaticLibrary -else: builder = env.SharedLibrary - -if (not env_etc.no_boost_python): - Import("env_boost_python_ext") - env_local_ext = env_boost_python_ext.Clone() - env_local_ext.Prepend( - LIBS=["cctbx", "scitbx_boost_python"]) - env_local_ext.SharedLibrary( - target="#lib/prime_index_ambiguity_ext", source="ext.cpp") - - env_etc.include_registry.append( - env=env_local_ext, - paths=env_etc.prime_common_includes) diff --git a/prime/index_ambiguity/__init__.py b/prime/index_ambiguity/__init__.py deleted file mode 100644 index abbc81f5dcb..00000000000 --- a/prime/index_ambiguity/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from __future__ import absolute_import, division, print_function -import boost_adaptbx.boost.python as bp -ext = bp.import_ext("prime_index_ambiguity_ext") -from prime_index_ambiguity_ext import * diff --git a/prime/index_ambiguity/ext.cpp b/prime/index_ambiguity/ext.cpp deleted file mode 100644 index cec9f8e0137..00000000000 --- a/prime/index_ambiguity/ext.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include -#include -#include - -namespace prime { namespace index_ambiguity { - scitbx::af::shared - calc_BD_alg_2(scitbx::af::versa > r_grid, scitbx::af::shared > x_vec_set) { - //calculate residuals of the Brehm & Diederichs Algorithm 2 - scitbx::af::shared r_out; - for (int i = 0; i < x_vec_set.size() - 1; i++) { - double sum_r = 0; - for (int j = i+1; j < x_vec_set.size(); j++) { - sum_r += std::abs(r_grid(i,j) - (x_vec_set[i] * x_vec_set[j])); - } - r_out.push_back(sum_r); - } - return r_out; - } - double - calc_BD_alg_2_sum_sqr(scitbx::af::versa > r_grid, scitbx::af::shared > x_vec_set) { - //calculate squared residuals of the Brehm & Diederichs Algorithm 2 - double sum_r_sqr = 0; - for (int i = 0; i < x_vec_set.size() - 1; i++) { - for (int j = i+1; j < x_vec_set.size(); j++) { - double tmp = r_grid(i,j) - (x_vec_set[i] * x_vec_set[j]); - sum_r_sqr += (tmp * tmp); - } - } - return sum_r_sqr; - } - scitbx::af::shared - calc_gradient_BD_alg_2(scitbx::af::versa > r_grid, scitbx::af::shared > x_vec_set) { - //calculate finite differences of each pair of parameters - scitbx::af::shared g_out; - double f = calc_BD_alg_2_sum_sqr(r_grid, x_vec_set); - double delta = 1.0e-7; - for (int i = 0; i < x_vec_set.size(); i++) { - //each component of the vector has its own gradient - x_vec_set[i][0] += delta; - double df_x = calc_BD_alg_2_sum_sqr(r_grid, x_vec_set); - g_out.push_back((df_x - f)/delta); - x_vec_set[i][0] -= delta; - x_vec_set[i][1] += delta; - double df_y = calc_BD_alg_2_sum_sqr(r_grid, x_vec_set); - g_out.push_back((df_y - f)/delta); - x_vec_set[i][1] -= delta; - } - return g_out; - } -}} - -namespace prime { namespace index_ambiguity { -namespace boost_python { - void - wrap_all() { - boost::python::def("calc_BD_alg_2", calc_BD_alg_2); - boost::python::def("calc_BD_alg_2_sum_sqr", calc_BD_alg_2_sum_sqr); - boost::python::def("calc_gradient_BD_alg_2", calc_gradient_BD_alg_2); - } -}}} - -BOOST_PYTHON_MODULE(prime_index_ambiguity_ext) -{ - prime::index_ambiguity::boost_python::wrap_all(); -} diff --git a/prime/index_ambiguity/mod_indexing_ambiguity.py b/prime/index_ambiguity/mod_indexing_ambiguity.py deleted file mode 100644 index d41d9201f3c..00000000000 --- a/prime/index_ambiguity/mod_indexing_ambiguity.py +++ /dev/null @@ -1,142 +0,0 @@ -from __future__ import absolute_import, division, print_function -from six.moves import cPickle as pickle -from prime.postrefine import postref_handler -from .mod_lbfgs import lbfgs_handler -import numpy as np -from cctbx import sgtbx -import random -from cctbx.array_family import flex -from prime.postrefine.mod_input import read_frame -from six.moves import range -from six.moves import zip - -class indamb_handler(object): - """ - handle indexing ambiguity main - """ - def __init__(self): - """ - Constructor - """ - - def generate_twin_operators(self, obs_in, flag_all=False): - #generate only true merohedral twin operators - from mmtbx.scaling.twin_analyses import twin_laws - TL = twin_laws(miller_array=obs_in) - operators = [] - if flag_all: - operators = TL.operators - else: - if TL.m > 0: - operators = TL.operators - return operators - - def generate_reindex_sets(self, obs_in): - ops = self.generate_twin_operators(obs_in) - alternates = {'h,k,l': obs_in} - for op in ops: - hkl = obs_in.indices() - cb_op = sgtbx.change_of_basis_op(op.operator.r().as_hkl()) - hklrev = cb_op.apply(hkl) - alternates[op.operator.r().as_hkl()] = obs_in.customized_copy(indices = hklrev).map_to_asu() - return alternates - - def generate_forced_reindex_sets(self, obs_in, assigned_basis): - alternates = {'h,k,l': obs_in} - for ab in assigned_basis: - cb_op = sgtbx.change_of_basis_op(ab) - alternates[ab] = obs_in.change_basis(cb_op).map_to_asu() - return alternates - - def get_observations(self, pickle_filename, iparams): - main_obs_pickle = read_frame(pickle_filename) - prh = postref_handler() - avg_mode = 'average' - try: - inputs, txt_org = prh.organize_input(main_obs_pickle, iparams, avg_mode, pickle_filename=pickle_filename) - main_obs = inputs[0] - except Exception: - print('Error reading input pickle.') - return None - main_asu = main_obs.map_to_asu().merge_equivalents().array() - #get other indexing alternatives - if iparams.indexing_ambiguity.mode == 'Forced': - #generate other basis format according to the list - alternates = self.generate_forced_reindex_sets(main_asu, iparams.indexing_ambiguity.assigned_basis) - else: - alternates = self.generate_reindex_sets(main_asu) - return alternates - - def calc_cc(self, frame_no, pickle_filename, iparams, miller_array_ref): - img_filename_only = '' - pickle_filepaths = pickle_filename.split('/') - img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] - try: - alternates = self.get_observations(pickle_filename, iparams) - cc_set = [] - for key in alternates.keys(): - corr = miller_array_ref.correlation(alternates[key], assert_is_similar_symmetry=False) - cc_set.append(corr.coefficient()) - i_best = np.argmax(cc_set) - txt_out = ' {0:40} ==> CC:{1:6.2f}'.format(img_filename_only+' '+list(alternates.keys())[i_best], cc_set[i_best]) - return list(alternates.keys())[i_best], txt_out - except Exception: - txt_out = ' {0:40} ==> CC:{1:6.2f}'.format(img_filename_only+' (h,k,l)', 0) - return "h,k,l", txt_out - - def calc_r(self, frame_no, pickle_filename, index_basis, main_obs, obs_results): - #get main observation - img_filename_only = '' - pickle_filepaths = pickle_filename.split('/') - img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] - txt_out = ' {0:40} ==> '.format(img_filename_only+' '+index_basis) - #calculate cc with other observations - n_frames = len(obs_results) - r_set = np.zeros((1, n_frames)) - n_refl_common_set = np.zeros((1, n_frames)) - n_frame_common = 0 - for j in range(frame_no+1, n_frames): - obs_asu = obs_results[j] - corr = main_obs.correlation(obs_asu, assert_is_similar_symmetry=False) - c_main, c_other = main_obs.common_sets(obs_asu, assert_is_similar_symmetry=False) - if len(c_main.indices()) > 5: - r_set[0,j] = corr.coefficient() - n_refl_common_set[0,j] = len(c_main.indices()) - n_frame_common += 1 - txt_out += ' :%6.2f : %6.1f N_frame_common: %6.0f'%(np.mean(r_set), np.mean(n_refl_common_set), n_frame_common) - return r_set, txt_out - - def optimize(self, r_matrix, flag_plot=False): - xinp = flex.double([random.random() for i in range(len(r_matrix)*2)]) - xinp_copy = xinp[:] - args = r_matrix - lh = lbfgs_handler(current_x=xinp, args=args) - x_set = np.array(lh.x).reshape((len(lh.x)//2,2)) - if flag_plot: - import matplotlib.pyplot as plt - xinp_set = np.array(xinp_copy).reshape((len(xinp_copy)/2,2)) - ax1 = plt.subplot(2,1,1) - ax1.scatter(xinp_set[:,0], xinp_set[:,1], s=10, marker='o', c='b') - ax1.set_xlim([-0.2, 1.2]) - ax1.set_ylim([-0.2, 1.2]) - ax2 = plt.subplot(2,1,2) - ax2.scatter(x_set[:,0], x_set[:,1], s=10, marker='o', c='b') - ax2.set_xlim([-0.2, 1.2]) - ax2.set_ylim([-0.2, 1.2]) - plt.show() - return x_set - - def assign_basis(self, frame_dup_files, basis_choices, labels, k, sample_fname): - sol_dict = {} - file_list_txt = '' - for i in range(k): - frame_cluster = np.extract(labels==i, frame_dup_files) - basis_choice_cluster = np.extract(labels==i, basis_choices) - for frame, basis_choice in zip(frame_cluster, basis_choice_cluster): - if frame not in sol_dict: - sol_dict[frame] = basis_choice - file_list_txt += frame+'\n' - f = open(sample_fname, 'w') - f.write(file_list_txt) - f.close() - return sol_dict diff --git a/prime/index_ambiguity/mod_kmeans.py b/prime/index_ambiguity/mod_kmeans.py deleted file mode 100644 index 7ab2b645a76..00000000000 --- a/prime/index_ambiguity/mod_kmeans.py +++ /dev/null @@ -1,96 +0,0 @@ -from __future__ import absolute_import, division, print_function -from six.moves import range -from six.moves import zip -''' -Author : Uervirojnangkoorn, M. -Created : 8/17/2016 -Description : Handy class to do K-means clustering. -''' -import random -import numpy as np -import matplotlib.pyplot as plt -from random import randint - -class kmeans_handler(object): - """ - From a given N-dimensional array, find K clusters. - Algorithms taken from Stanford CS221 pseudo-python code. - See http://stanford.edu/~cpiech/cs221/handouts/kmeans.html - """ - def __init__(self): - """ - Constructor - """ - self.MAX_ITERS = 1000 - - def run(self, dataset, k, flag_plot=False): - #define color for plotting - colors = ['#%06X' % randint(0, 0xFFFFFF) for i in range(k)] - #initialize controids randomly - n_data, n_features = dataset.shape - centroids = np.array([random.random() for i in range(k*n_features)]).reshape((k,n_features)) - scale = np.max(dataset, axis=0) - centroids = centroids * scale - if flag_plot: - plt.scatter(dataset[:,0], dataset[:,1], s=10, marker='x', c='b') - plt.scatter(centroids[:,0], centroids[:,1], s=20, marker='o', c='k') - plt.title('Initial step (k=%3.0f)'%(k)) - plt.show() - #initialize book keeping vars - n_iters = 0 - old_centroids = None - #run the k-means algorithm - while not self.should_stop(old_centroids, centroids, n_iters): - #save old centroids for convergenence test. book keeping. - old_centroids = centroids[:] - n_iters += 1 - #assign labels to each datapoint based on centroids - labels = self.get_labels(dataset, centroids) - #assign centroids based on datapoint labels - centroids = self.get_centroids(dataset, labels, k) - if flag_plot: - for i in range(k): - s = dataset[labels==i] - plt.scatter(s[:,0], s[:,1], s=10, marker='x', c=colors[i]) - plt.scatter(centroids[i,0], centroids[i,1], s=20, marker='o', c='k') - plt.title('Cycle %3.0f (k=%3.0f)'%(n_iters, k)) - plt.show() - return centroids, labels - - def should_stop(self, old_centroids, centroids, n_iters): - if n_iters > self.MAX_ITERS: return True - return np.all(old_centroids == centroids) - - def get_labels(self, dataset, centroids): - #For each element in the dataset, choose the closest centroid. - #Make that centroid the element's label. - n_data, n_features = dataset.shape - n_centroids = len(centroids) - dist = np.zeros((n_data, n_centroids)) - for c,i in zip(centroids, range(n_centroids)): - dist[:,i] = np.sqrt(((dataset-c)**2).sum(axis=1)) - labels = np.argmin(dist, axis=1) - return labels - - def get_centroids(self, dataset, labels, k): - #For each label group, calculate a new geometric mean as a new centroid. - #For empty group, assign a new random centroid. - n_data, n_features = dataset.shape - centroids = np.zeros((k, n_features)) - for i in range(k): - s = dataset[labels==i] - if len(s) > 0: - centroids[i] = np.power(np.sqrt(np.prod(s, axis=0)**2), 1/len(s)) - else: - scale = np.max(dataset, axis=0) - centroids[i] = np.array([random.random() for i in range(n_features)]) * scale - return centroids - -if __name__=="__main__": - #unit test - points = np.vstack([np.random.multivariate_normal(mean, \ - 0.03 * np.diag([1,1]), 20) \ - for mean in [(1, 1), (2, 4), (3, 2)]]) - k = 3 - kmh = kmeans_handler() - centroids, labels = kmh.run(points, k, flag_plot=True) diff --git a/prime/index_ambiguity/mod_lbfgs.py b/prime/index_ambiguity/mod_lbfgs.py deleted file mode 100644 index 882fcc4a468..00000000000 --- a/prime/index_ambiguity/mod_lbfgs.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import absolute_import, division, print_function -from scitbx import lbfgs -from cctbx.array_family import flex -from prime import index_ambiguity -import numpy as np - -class lbfgs_handler(object): - """ - lbfgs_handler optimize set of parameters (params) by fitting - data[0] to data [1] using given function (func). - """ - def __init__(self, current_x=None, args=None, - min_iterations=0, max_iterations=None, max_calls=1000, max_drop_eps=1.e-5): - self.n = current_x.size() - self.x = current_x - self.args = args - self.minimizer = lbfgs.run( - target_evaluator=self, - termination_params=lbfgs.termination_parameters( - traditional_convergence_test=False, - drop_convergence_test_max_drop_eps=max_drop_eps, - min_iterations=min_iterations, - max_iterations=max_iterations, - max_calls=max_calls), - exception_handling_params=lbfgs.exception_handling_parameters( - ignore_line_search_failed_rounding_errors=True, - ignore_line_search_failed_step_at_lower_bound=True, - ignore_line_search_failed_step_at_upper_bound=False, - ignore_line_search_failed_maxfev=False, - ignore_line_search_failed_xtol=False, - ignore_search_direction_not_descent=False) - ) - - def compute_functional_and_gradients(self): - x_set = np.array(self.x).reshape((len(self.x)//2,2)) - r_grid = flex.double(self.args) - x_vec_set = flex.vec2_double(x_set) - self.f = index_ambiguity.calc_BD_alg_2_sum_sqr(r_grid, x_vec_set) - self.g = index_ambiguity.calc_gradient_BD_alg_2(r_grid, x_vec_set) - return self.f, self.g diff --git a/prime/isoform_cluster/__init__.py b/prime/isoform_cluster/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/prime/isoform_cluster/mod_isoform_cluster.py b/prime/isoform_cluster/mod_isoform_cluster.py deleted file mode 100644 index 4b84f8bf685..00000000000 --- a/prime/isoform_cluster/mod_isoform_cluster.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import absolute_import, division, print_function -from prime.postrefine import postref_handler -from prime.index_ambiguity.mod_lbfgs import lbfgs_handler -from cctbx.array_family import flex -import numpy as np -import random, ntpath, os -from six.moves import range - -class isoform_cluster_handler(object): - """ - handle indexing ambiguity main - """ - def get_observations(self, pickle_filename, iparams): - prh = postref_handler() - #modify reflection filters - iparams.merge.d_min = iparams.isoform_cluster.d_min - iparams.merge.d_max = iparams.isoform_cluster.d_max - iparams.merge.sigma_min = iparams.isoform_cluster.sigma_min - pres, txt_out = prh.scale_frame_by_mean_I(-1, pickle_filename, iparams, 0, 'average') - return pres.get_full_observations().merge_equivalents().array() if pres else None - - def calc_cc(self, frame_no, pickle_filename, main_obs, iparams, miller_array_ref_set): - img_filename_only = ntpath.basename(pickle_filename) - cc_set = [] - for miller_array_ref in miller_array_ref_set: - corr = main_obs.correlation(miller_array_ref, assert_is_similar_symmetry=False) - cc_set.append(corr.coefficient()) - i_best = np.argmax(cc_set) - txt_out = ' {0:40} {1:2d} ==> CC:{2:6.2f} '.format(img_filename_only, i_best, cc_set[i_best]) - return i_best, txt_out - - def calc_r(self, frame_no, pickle_filename, main_obs, obs_results): - #get main observation - img_filename_only = ntpath.basename(pickle_filename) - txt_out = ' {0:40} ==> '.format(img_filename_only) - #calculate cc with other observations - n_frames = len(obs_results) - r_set = np.zeros((1, n_frames)) - n_refl_common_set = np.zeros((1, n_frames)) - n_frame_common = 0 - for j in range(frame_no+1, n_frames): - obs_asu = obs_results[j] - corr = main_obs.correlation(obs_asu, assert_is_similar_symmetry=False) - c_main, c_other = main_obs.common_sets(obs_asu, assert_is_similar_symmetry=False) - if len(c_main.indices()) > 5: - r_set[0,j] = corr.coefficient() - n_refl_common_set[0,j] = len(c_main.indices()) - n_frame_common += 1 - txt_out += ' :%6.2f : %6.1f N_frame_common: %6.0f'%(np.mean(r_set), np.mean(n_refl_common_set), n_frame_common) - return r_set, main_obs, txt_out - - def optimize(self, r_matrix, flag_plot=False): - xinp = flex.double([random.random() for i in range(len(r_matrix)*2)]) - xinp_copy = xinp[:] - args = r_matrix - lh = lbfgs_handler(current_x=xinp, args=args) - x_set = np.array(lh.x).reshape((len(lh.x)/2,2)) - if flag_plot: - import matplotlib.pyplot as plt - xinp_set = np.array(xinp_copy).reshape((len(xinp_copy)/2,2)) - plt.subplot(2,1,1) - plt.scatter(xinp_set[:,0], xinp_set[:,1], s=10, marker='x', c='r') - plt.subplot(2,1,2) - plt.scatter(x_set[:,0], x_set[:,1], s=10, marker='x', c='r') - plt.show() - return x_set - - def assign_cluster(self, frame_files, labels, k, output_path): - sol_dict = {} - cluster_files = [os.path.join(output_path,'cluster_'+str(i)+'.lst') for i in range(k)] - for i in range(k): - file_list_txt = '' - frame_cluster = np.extract(labels==i, frame_files) - for frame in frame_cluster: - if frame not in sol_dict: - sol_dict[frame] = i - file_list_txt += frame+'\n' - with open(cluster_files[i], 'w') as f: f.write(file_list_txt) - return sol_dict, cluster_files diff --git a/prime/libtbx_config b/prime/libtbx_config deleted file mode 100644 index 4ffdce6c007..00000000000 --- a/prime/libtbx_config +++ /dev/null @@ -1,3 +0,0 @@ -{ - "modules_required_for_use": ["xfel"], -} diff --git a/prime/postrefine/__init__.py b/prime/postrefine/__init__.py deleted file mode 100644 index cd41890bd80..00000000000 --- a/prime/postrefine/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from __future__ import absolute_import, division, print_function -from .postrefine import * diff --git a/prime/postrefine/mod_gui_dialogs.py b/prime/postrefine/mod_gui_dialogs.py deleted file mode 100644 index 5722546d20c..00000000000 --- a/prime/postrefine/mod_gui_dialogs.py +++ /dev/null @@ -1,705 +0,0 @@ -from __future__ import division, print_function, absolute_import - -''' -Author : Lyubimov, A.Y. -Created : 05/01/2016 -Last Changed: 07/17/2019 -Description : PRIME GUI dialogs module -''' - -import getpass -import os -import wx -from wx.lib.scrolledpanel import ScrolledPanel -from wxtbx import bitmaps - -from iotbx import phil as ip - -from iota.utils.utils import WxFlags, Capturing -from iota.gui.base import BaseBackendDialog, IOTABaseDialog -import iota.gui.controls as ct - -# Platform-specific stuff -# TODO: Will need to test this on Windows at some point -if wx.Platform == '__WXGTK__': - norm_font_size = 10 - button_font_size = 12 - LABEL_SIZE = 14 - CAPTION_SIZE = 12 - python = 'python' -elif wx.Platform == '__WXMAC__': - norm_font_size = 12 - button_font_size = 14 - LABEL_SIZE = 14 - CAPTION_SIZE = 12 - python = "Python" -elif (wx.Platform == '__WXMSW__'): - norm_font_size = 9 - button_font_size = 11 - LABEL_SIZE = 11 - CAPTION_SIZE = 9 - -f = WxFlags() -user = getpass.getuser() - - -class PRIMEBaseBackendDialog(BaseBackendDialog): - def __init__(self, parent, - content_style='normal', - label_style='bold', - *args, **kwargs): - BaseBackendDialog.__init__(self, parent, - content_style=content_style, - label_style=label_style, - *args, **kwargs) - - def get_target_file(self): - dlg = wx.FileDialog( - self, message="Select PRIME settings file", - defaultDir=os.curdir, - defaultFile="*.phil", - wildcard="*", - style=wx.FD_OPEN | wx.FD_CHANGE_DIR - ) - if dlg.ShowModal() == wx.ID_OK: - filepath = dlg.GetPaths()[0] - - with open(filepath, 'r') as phil_file: - phil_content = phil_file.read() - return phil_content - else: - return None - -def str_split(string, delimiters=(' ', ','), maxsplit=0): - import re - rexp = '|'.join(map(re.escape, delimiters)) - return re.split(rexp, string, maxsplit) - - -class PRIMEAdvancedOptions(PRIMEBaseBackendDialog): - ''' Advanced Options Dialog''' - - def __init__(self, parent, phil=None, *args, **kwargs): - PRIMEBaseBackendDialog.__init__(self, parent, - backend_name='PRIME', - phil=phil, - phil_size=(500, 500), - opt_size=(500, 500), - *args, **kwargs) - - self.prime_phil = phil - self.new_prime_phil = None - self.pparams = self.prime_phil.extract() - - self.splitter.SplitVertically(self.options, self.phil_panel) - - # Target file input - self.phil = ct.PHILBox(self.phil_panel, - btn_import=True, - btn_import_label='Import PHIL', - btn_export=False, - btn_default=False, - btn_pos='bottom', - ctr_size=(-1, 300), - ctr_value='') - self.phil_sizer.Add(self.phil, 1, flag=wx.EXPAND | wx.ALL, border=5) - - # PRIME Options (there is some redundancy with the PHIL textbox) - self.prm_options = wx.Panel(self.options) - opt_box = wx.StaticBox(self.prm_options, label='Advanced Options') - opt_box_sizer = wx.StaticBoxSizer(opt_box, wx.VERTICAL) - self.prm_options.SetSizer(opt_box_sizer) - - # Resolution - self.scale_res = ct.OptionCtrl(self.prm_options, - label='Scale Resolution: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.scale_res, flag=f.stack, border=10) - - self.merge_res = ct.OptionCtrl(self.prm_options, - label='Merge Resolution: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.merge_res, flag=f.stack, border=10) - - self.p_scale_res = ct.OptionCtrl(self.prm_options, - label='Postref Scale Res.: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.p_scale_res, flag=f.stack, border=10) - - self.p_cryst_res = ct.OptionCtrl(self.prm_options, - checkbox=True, - checkbox_label='Crystal Orientation: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.p_cryst_res, flag=f.stack, border=10) - - self.p_rrange_res = ct.OptionCtrl(self.prm_options, - checkbox=True, - checkbox_label='Reflecting Range: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.p_rrange_res, flag=f.stack, border=10) - - self.p_uc_res = ct.OptionCtrl(self.prm_options, - checkbox=True, - checkbox_label='Unit Cell: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.p_uc_res, flag=f.stack, border=10) - - self.p_all_res = ct.OptionCtrl(self.prm_options, - checkbox=True, - checkbox_label='All Parameters: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=wx.DefaultSize, - items=[('low', 50), - ('high', 1.5)]) - opt_box_sizer.Add(self.p_all_res, flag=f.stack, border=10) - - self.btn_synch_res = wx.Button(self.prm_options, - label="Set Res. Limits to Postref. Scale") - opt_box_sizer.Add(self.btn_synch_res, flag=f.stack, border=10) - - - # Target space group - self.sg = ct.OptionCtrl(self.prm_options, - label='Space Group: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=(100, -1), - items=[('spacegroup','P212121')]) - opt_box_sizer.Add(self.sg, flag=f.stack, border=10) - - # Target unit cell - self.uc = ct.OptionCtrl(self.prm_options, - label='Unit Cell: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=(300, -1), - items=[('unit_cell', '72 120 134 90 90 90')]) - self.anom = wx.CheckBox(self.prm_options, label='Anomalous') - self.anom.SetValue(False) - opt_box_sizer.Add(self.uc, flag=f.stack, border=10) - opt_box_sizer.Add(self.anom, flag=f.stack, border=10) - - # CC cutoff - self.cc = ct.OptionCtrl(self.prm_options, - label='CC cutoff: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=(100, -1), - items=[('cc_cutoff', 0.25)]) - opt_box_sizer.Add(self.cc, flag=f.stack, border=10) - - # Pixel size - self.pix = ct.OptionCtrl(self.prm_options, - label='Pixel size: ', - label_size=(120, -1), - label_style='normal', - ctrl_size=(100, -1), - items=[('pixel_size', 0.172)]) - opt_box_sizer.Add(self.pix, flag=wx.RIGHT | wx.LEFT | wx.TOP, border=10) - - self.cycles = ct.SpinCtrl(self.prm_options, - label='No. of Cycles:', - label_size=(120, -1), - label_style='normal', - ctrl_size=(60, -1)) - opt_box_sizer.Add(self.cycles, flag=wx.ALL, border=10) - - # self.options_sizer.Add(self.phil_sizer, 1, flag=wx.EXPAND | wx.ALL, border=10) - self.options_sizer.Add(self.prm_options, 1, flag=wx.EXPAND | wx.ALL, border=10) - - # Button bindings - self.Bind(wx.EVT_BUTTON, self.onImportPHIL, self.phil.btn_import) - self.Bind(wx.EVT_BUTTON, self.onOK, id=wx.ID_OK) - self.Bind(wx.EVT_BUTTON, self.onHideScript, self.btn_hide_script) - self.Bind(wx.EVT_BUTTON, self.onResSynchronize, self.btn_synch_res) - self.Bind(wx.EVT_CHOICE, self.onAdvanced, self.dlg_ctr.choice) - - self.show_hide_script() - self.show_hide_advanced(show=False) - self.Layout() - self.options.SetupScrolling() - self.read_param_phil() - - def onAdvanced(self, e): - mode = self.dlg_ctr.choice.GetSelection() - if mode == 0: - self.show_hide_advanced(show=False) - else: - self.show_hide_advanced(show=True) - - def show_hide_advanced(self, show=False): - if show: - self.p_all_res.Show() - self.p_cryst_res.Show() - self.p_rrange_res.Show() - self.p_scale_res.Show() - self.p_uc_res.Show() - self.btn_synch_res.Show() - else: - self.p_all_res.Hide() - self.p_cryst_res.Hide() - self.p_rrange_res.Hide() - self.p_scale_res.Hide() - self.p_uc_res.Hide() - self.btn_synch_res.Hide() - - self.options.Layout() - self.options.SetupScrolling() - - def read_param_phil(self): - # Put in PHIL script - self.generate_phil_string() - self.phil.ctr.SetValue(self.phil_string) - - # Set values to default parameters - self.scale_res.low.SetValue( - '{:4.2f}'.format(self.pparams.scale.d_max)) - self.scale_res.high.SetValue( - '{:4.2f}'.format(self.pparams.scale.d_min)) - self.merge_res.low.SetValue( - '{:4.2f}'.format(self.pparams.merge.d_max)) - self.merge_res.high.SetValue( - '{:4.2f}'.format(self.pparams.merge.d_min)) - self.p_scale_res.low.SetValue( - '{:4.2f}'.format(self.pparams.postref.scale.d_max)) - self.p_scale_res.high.SetValue( - '{:4.2f}'.format(self.pparams.postref.scale.d_min)) - self.p_cryst_res.toggle_boxes(flag_on=self.pparams.postref.crystal_orientation.flag_on) - self.p_cryst_res.low.SetValue( - '{:4.2f}'.format(self.pparams.postref.crystal_orientation.d_max)) - self.p_cryst_res.high.SetValue( - '{:4.2f}'.format(self.pparams.postref.crystal_orientation.d_min)) - self.p_rrange_res.toggle_boxes(flag_on=self.pparams.postref.reflecting_range.flag_on) - self.p_rrange_res.low.SetValue( - '{:4.2f}'.format(self.pparams.postref.reflecting_range.d_max)) - self.p_rrange_res.high.SetValue( - '{:4.2f}'.format(self.pparams.postref.reflecting_range.d_min)) - self.p_uc_res.toggle_boxes(flag_on=self.pparams.postref.unit_cell.flag_on) - self.p_uc_res.low.SetValue( - '{:4.2f}'.format(self.pparams.postref.unit_cell.d_max)) - self.p_uc_res.high.SetValue( - '{:4.2f}'.format(self.pparams.postref.unit_cell.d_min)) - self.p_all_res.toggle_boxes(flag_on=self.pparams.postref.allparams.flag_on) - self.p_all_res.low.SetValue( - '{:4.2f}'.format(self.pparams.postref.allparams.d_max)) - self.p_all_res.high.SetValue( - '{:4.2f}'.format(self.pparams.postref.allparams.d_min)) - - self.sg.spacegroup.SetValue(str(self.pparams.target_space_group)) - if str(self.pparams.target_unit_cell).lower() != 'none': - uc = ' '.join(list(map(str, self.pparams.target_unit_cell.parameters()))) - self.uc.unit_cell.SetValue(uc) - else: - self.uc.unit_cell.SetValue(str(self.pparams.target_unit_cell)) - self.anom.SetValue(self.pparams.target_anomalous_flag) - self.cc.cc_cutoff.SetValue(str(self.pparams.frame_accept_min_cc)) - self.pix.pixel_size.SetValue(str(self.pparams.pixel_size_mm)) - self.cycles.ctr.SetValue(int(self.pparams.n_postref_cycle)) - - def generate_phil_string(self): - with Capturing() as txt_output: - self.prime_phil.show() - self.phil_string = '' - for one_output in txt_output: - self.phil_string += one_output + '\n' - - def onHideScript(self, e): - self.opt_size = self.options.GetSize() - self.phil_size = self.phil_panel.GetSize() - self.sash_position = self.opt_size[0] - self.show_hide_script(initialized=True) - - def onImportPHIL(self, e): - phil_content = self.get_target_file() - if phil_content is not None: - self.phil.ctr.SetValue(phil_content) - - def onResSynchronize(self, e): - self.synchronize_resolution() - - def synchronize_resolution(self): - # Set all activated resolution ranges to Merge Resolution - scale_low = self.merge_res.low.GetValue() - scale_high = self.merge_res.high.GetValue() - self.p_scale_res.low.SetValue(scale_low) - self.p_scale_res.high.SetValue(scale_high) - if self.p_cryst_res.toggle.GetValue(): - self.p_cryst_res.low.SetValue(scale_low) - self.p_cryst_res.high.SetValue(scale_high) - if self.p_rrange_res.toggle.GetValue(): - self.p_rrange_res.low.SetValue(scale_low) - self.p_rrange_res.high.SetValue(scale_high) - if self.p_uc_res.toggle.GetValue(): - self.p_uc_res.low.SetValue(scale_low) - self.p_uc_res.high.SetValue(scale_high) - if self.p_all_res.toggle.GetValue(): - self.p_all_res.low.SetValue(scale_low) - self.p_all_res.high.SetValue(scale_high) - - def onOK(self, e): - self.phil_script = ip.parse(self.phil.ctr.GetValue()) - - prime_phil_text = '\n'.join([ - 'scale', - '{', - ' d_min = {}'.format(float(self.scale_res.high.GetValue())), - ' d_max = {}'.format(float(self.scale_res.low.GetValue())), - '}', - 'postref', - '{', - ' scale', - ' {', - ' d_min = {}'.format(float(self.p_scale_res.high.GetValue())), - ' d_max = {}'.format(float(self.p_scale_res.low.GetValue())), - ' }', - ' crystal_orientation', - ' {', - ' flag_on = {}'.format(self.p_cryst_res.toggle.GetValue()), - ' d_min = {}'.format(float(self.p_cryst_res.high.GetValue())), - ' d_max = {}'.format(float(self.p_cryst_res.low.GetValue())), - ' }', - ' reflecting_range', - ' {', - ' flag_on = {}'.format(self.p_rrange_res.toggle.GetValue()), - ' d_min = {}'.format(float(self.p_rrange_res.high.GetValue())), - ' d_max = {}'.format(float(self.p_rrange_res.low.GetValue())), - ' }', - ' unit_cell', - ' {', - ' flag_on = {}'.format(self.p_uc_res.toggle.GetValue()), - ' d_min = {}'.format(float(self.p_uc_res.high.GetValue())), - ' d_max = {}'.format(float(self.p_uc_res.low.GetValue())), - ' }', - ' allparams', - ' {', - ' flag_on = {}'.format(self.p_all_res.toggle.GetValue()), - ' d_min = {}'.format(float(self.p_all_res.high.GetValue())), - ' d_max = {}'.format(float(self.p_all_res.low.GetValue())), - ' }', - '}', - 'merge', - '{', - ' d_min = {}'.format(float(self.merge_res.high.GetValue())), - ' d_max = {}'.format(float(self.merge_res.low.GetValue())), - '}', - 'target_unit_cell = {}'.format(self.uc.unit_cell.GetValue()), - 'target_space_group = {}'.format(self.sg.spacegroup.GetValue()), - 'target_anomalous_flag = {}'.format(self.anom.GetValue()), - 'pixel_size_mm = {}'.format(self.pix.pixel_size.GetValue()), - 'frame_accept_min_cc = {}'.format(self.cc.cc_cutoff.GetValue()), - 'n_postref_cycle = {}'.format(self.cycles.ctr.GetValue()) - ]) - - self.new_prime_phil = ip.parse(prime_phil_text) - e.Skip() - -class PRIMEPreferences(IOTABaseDialog): - def __init__(self, parent, phil=None, *args, **kwargs): - IOTABaseDialog.__init__(self, parent, *args, **kwargs) - - self.pparams = phil.extract() - self.pref_phil = None - - main_sizer = wx.BoxSizer(wx.VERTICAL) - - main_box = wx.StaticBox(self, label='PRIME Preferences') - vbox = wx.StaticBoxSizer(main_box, wx.VERTICAL) - - self.SetSizer(main_sizer) - - q_choices = ['psanaq', 'psnehq', 'psfehq'] + ['custom'] - self.queues = ct.ChoiceCtrl(self, - label='Queue:', - label_size=(120, -1), - label_style='bold', - ctrl_size=wx.DefaultSize, - choices=q_choices) - vbox.Add(self.queues, flag=wx.ALL, border=10) - - self.custom_queue = ct.OptionCtrl(self, - items=[('cqueue', '')], - label='Custom Queue:', - label_size=(120, -1), - label_style='normal', - ctrl_size=(150, -1)) - self.custom_queue.Disable() - vbox.Add(self.custom_queue, flag=wx.ALL, border=10) - - self.nodes = ct.SpinCtrl(self, - label='No. of nodes', - label_size=(120, -1), - ctrl_value=12, - ctrl_size=(80, -1), - ctrl_min=1, - ctrl_max=5000) - self.nodes.Disable() - vbox.Add(self.nodes, flag=wx.ALL, border=10) - - mp_choices = ['multiprocessing', 'bsub'] - self.mp_methods = ct.ChoiceCtrl(self, - label='Method:', - label_size=(120, -1), - label_style='bold', - ctrl_size=wx.DefaultSize, - choices=mp_choices) - vbox.Add(self.mp_methods, flag=wx.ALL, border=10) - - main_sizer.Add(vbox, flag=wx.EXPAND | wx.ALL, border=10) - - # Dialog control - dialog_box = self.CreateSeparatedButtonSizer(wx.OK | wx.CANCEL) - main_sizer.Add(dialog_box, - flag=wx.EXPAND | wx.ALIGN_RIGHT | wx.ALL, - border=10) - - self.Bind(wx.EVT_CHOICE, self.onQueue, self.queues.ctr) - self.Bind(wx.EVT_CHOICE, self.onMethod, self.mp_methods.ctr) - self.Bind(wx.EVT_BUTTON, self.onOK, id=wx.ID_OK) - - self.set_choices() - - def set_choices(self): - # Set queue to default value - queue = self.pparams.queue.qname - method = self.pparams.queue.mode - if queue is None: - queue = 'None' - inp_queue = self.queues.ctr.FindString(queue) - if inp_queue != wx.NOT_FOUND: - self.queues.ctr.SetSelection(inp_queue) - else: - self.custom_queue.Enable() - self.custom_queue.cqueue.SetValue(queue) - custom_queue = self.queues.ctr.FindString('custom') - self.queues.ctr.SetSelection(custom_queue) - - # Set method to default value - inp_method = self.mp_methods.ctr.FindString(str(method)) - if inp_method != wx.NOT_FOUND: - self.mp_methods.ctr.SetSelection(inp_method) - - self.nodes.ctr.SetValue(self.pparams.queue.n_nodes) - self.check_method() - - def onMethod(self, e): - self.check_method() - - def check_method(self): - choice = self.mp_methods.ctr.GetString(self.mp_methods.ctr.GetSelection()) - if choice == 'multiprocessing': - self.queues.Disable() - self.custom_queue.Disable() - else: - self.queues.Enable() - self.nodes.Enable() - queue = self.queues.ctr.GetString(self.queues.ctr.GetSelection()) - if queue == 'custom': - self.custom_queue.Enable() - - def onQueue(self, e): - choice = self.queues.ctr.GetString(self.queues.ctr.GetSelection()) - if choice == 'custom': - self.custom_queue.Enable() - else: - self.custom_queue.Disable() - - def onOK(self, e): - self.method = self.mp_methods.ctr.GetString(self.mp_methods.ctr.GetSelection()) - queue_selection = self.queues.ctr.GetString(self.queues.ctr.GetSelection()) - if queue_selection == 'custom': - if self.custom_queue.cqueue.GetValue() == '': - wx.MessageBox('Please choose or enter a queue', wx.OK) - else: - self.queue = self.custom_queue.cqueue.GetValue() - e.Skip() - else: - self.queue = queue_selection - e.Skip() - - if self.method != 'multiprocessing': - n_nodes = self.nodes.ctr.GetValue() - else: - n_nodes = 12 - - prime_prefs_string = '\n'.join([ - 'queue {', - ' mode = {}'.format(self.method), - ' qname = {}'.format(self.queue), - ' n_nodes = {}'.format(n_nodes), - '}' - ]) - - self.pref_phil = ip.parse(prime_prefs_string) - e.Skip() - - -class TextFileView(IOTABaseDialog): - def __init__(self, parent, - label_style='bold', - content_style='normal', - contents=None, - *args, **kwargs): - - dlg_style = wx.CAPTION | wx.CLOSE_BOX | wx.RESIZE_BORDER | wx.STAY_ON_TOP - - IOTABaseDialog.__init__(self, parent, style=dlg_style, - label_style=label_style, - content_style=content_style, - size=(600, 500), - *args, **kwargs) - - self.txt_panel = ScrolledPanel(self) - self.txt_sizer = wx.BoxSizer(wx.VERTICAL) - self.txt_panel.SetSizer(self.txt_sizer) - - self.txt = wx.StaticText(self.txt_panel, label=contents) - self.txt_sizer.Add(self.txt) - - self.txt_panel.SetupScrolling() - self.envelope.Add(self.txt_panel, 1, flag=wx.EXPAND | wx.ALL, border=10) - - -class RecoveryDialog(IOTABaseDialog): - - def __init__(self, - parent, - label_style='bold', - content_style='normal', - *args, **kwargs): - - dlg_style = wx.CAPTION | wx.CLOSE_BOX | wx.RESIZE_BORDER | wx.STAY_ON_TOP - IOTABaseDialog.__init__(self, parent, style=dlg_style, - label_style=label_style, - content_style=content_style, - size=(400, 400), - title='Recover Completed Run', - *args, **kwargs) - - self.pathlist = wx.ListCtrl(self, style=wx.LC_REPORT|wx.LC_SINGLE_SEL) - self.selected = None - self.recovery_mode = 0 - - self.pathlist.InsertColumn(0, "") - self.pathlist.InsertColumn(1, "Status") - self.pathlist.InsertColumn(2, "Path") - - bmps = wx.ImageList(width=16, height=16) - finished_bmp = bitmaps.fetch_icon_bitmap('actions', 'button_ok', size=16) - aborted_bmp = bitmaps.fetch_icon_bitmap('actions', 'cancel', size=16) - unknown_bmp = bitmaps.fetch_icon_bitmap('actions', 'status_unknown', - size=16) - bmps.Add(finished_bmp) - bmps.Add(aborted_bmp) - bmps.Add(unknown_bmp) - self.pathlist.AssignImageList(bmps, which=1) - - self.envelope.Add(self.pathlist, 1, flag=wx.EXPAND | wx.ALL, border=10) - - # Dialog control - self.dlg_ctr = ct.DialogButtonsCtrl(self, preset='OK_CANCEL', - choices=['everything', 'settings only'], - choice_label='Recover: ') - self.envelope.Add(self.dlg_ctr, flag=wx.EXPAND | wx.ALIGN_RIGHT | wx.ALL, - border=10) - - self.Bind(wx.EVT_BUTTON, self.onOK, id=wx.ID_OK) - self.Bind(wx.EVT_BUTTON, self.onCancel, self.dlg_ctr.btn_cancel) - self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.onItemActivated, id=-1) - - self.place_and_size(set_by='parent', set_size='dialog') - - def insert_paths(self, pathlist): - for i in range(len(pathlist)): - logfile = os.path.join(pathlist[i], 'log.txt') - if os.path.isfile(logfile): - with open(logfile, 'r') as lf: - log_contents = lf.readlines() - else: - log_contents = [] - img_id = 2 - status = "Unknown" - - stats_folder = os.path.join(pathlist[i], 'stats') - try: - stat_files = [os.path.join(stats_folder, f) for f in - os.listdir(stats_folder) if f.endswith('stat')] - except OSError: - stat_files = [] - - if stat_files != []: - assert len(stat_files) == 1 - stat_file = stat_files[0] - if log_contents != [] and 'Finished' in log_contents[-1]: - img_id = 0 - status = 'Finished' - elif log_contents != [] and os.path.isfile(stat_file): - img_id = 1 - status = 'Aborted' - else: - img_id = 2 - status = "Unknown" - - idx = self.pathlist.InsertImageItem(i, img_id) - self.pathlist.SetStringItem(idx, 1, status) - self.pathlist.SetStringItem(idx, 2, pathlist[i]) - self.pathlist.SetColumnWidth(0, width=-1) - self.pathlist.SetColumnWidth(1, width=-1) - self.pathlist.SetColumnWidth(2, width=-1) - - def set_selection(self, idx=0): - self.selected = [self.pathlist.GetItemText(idx, col=1), - self.pathlist.GetItemText(idx, col=2)] - self.recovery_mode = self.dlg_ctr.choice.GetSelection() - - def onOK(self, e): - selected = False - for i in range(self.pathlist.GetItemCount()): - if self.pathlist.IsSelected(i): - - self.selected = [self.pathlist.GetItemText(i, col=1), - self.pathlist.GetItemText(i, col=2)] - self.recovery_mode = self.dlg_ctr.choice.GetSelection() - selected = True - - if selected: - self.EndModal(wx.ID_OK) - else: - self.EndModal(wx.ID_CANCEL) - - e.Skip() - - def onCancel(self, e): - self.EndModal(wx.ID_CANCEL) - - def onItemActivated(self, e): - idx = e.GetIndex() - self.set_selection(idx=idx) - self.EndModal(wx.ID_OK) - e.Skip() - -# --- end diff --git a/prime/postrefine/mod_gui_frames.py b/prime/postrefine/mod_gui_frames.py deleted file mode 100644 index f22aef9dd44..00000000000 --- a/prime/postrefine/mod_gui_frames.py +++ /dev/null @@ -1,1502 +0,0 @@ -from __future__ import division, print_function, absolute_import - -''' -Author : Lyubimov, A.Y. -Created : 05/01/2016 -Last Changed: 12/02/2019 -Description : PRIME GUI frames module -''' - -import getpass -import os -import numpy as np -import multiprocessing - -import wx -from wxtbx import bitmaps - -from iotbx import phil as ip -from libtbx import easy_run, easy_pickle as ep -from libtbx.utils import Sorry - -import matplotlib as mpl -import matplotlib.gridspec as gridspec -from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas -from matplotlib.figure import Figure - -from iota.utils import utils -import iota.gui.controls as ct -import iota.gui.phil_controls as pct -from iota.gui import make_phil_index -from iota.gui.base import IOTABasePanel, IOTABaseFrame, IOTABaseScrolledPanel -from iota.gui.plotter import PlotWindow - -from prime import prime_license, prime_description -import prime.postrefine.mod_gui_dialogs as dlg -import prime.postrefine.mod_threads as thr -from prime.postrefine.mod_input import master_phil -from prime.postrefine.mod_plotter import Plotter - -user = getpass.getuser() -ginp = utils.InputFinder() -f = utils.WxFlags() - -# Platform-specific stuff -# TODO: Will need to test this on Windows at some point -if wx.Platform == '__WXGTK__': - plot_font_size = 9 - norm_font_size = 9 - button_font_size = 10 - big_button_font_size = 12 - LABEL_SIZE = 12 - CAPTION_SIZE = 10 - python = 'python' -elif wx.Platform == '__WXMAC__': - plot_font_size = 9 - norm_font_size = 12 - button_font_size = 14 - big_button_font_size = 16 - LABEL_SIZE = 14 - CAPTION_SIZE = 12 - python = "Python" -elif wx.Platform == '__WXMSW__': - plot_font_size = 9 - norm_font_size = 9 - button_font_size = 11 - big_button_font_size = 15 - LABEL_SIZE = 11 - CAPTION_SIZE = 9 - python = "Python" # TODO: make sure it's right! - -# def str_split(string, delimiters=(' ', ','), maxsplit=0): -# import re -# rexp = '|'.join(map(re.escape, delimiters)) -# return re.split(rexp, string, maxsplit) - - -# ------------------------------- Main Window ------------------------------ # - -class PRIMEInputWindow(IOTABasePanel): - ''' Main PRIME Window panel - - DEPRECATED! (Keeping it around while other apps still use it.) - ''' - - def __init__(self, parent, phil=None): - IOTABasePanel.__init__(self, parent=parent) - - self.regenerate_params(phil) - - # Title box - self.project_title = ct.InputCtrl(self, label='Project Title: ', - label_size=(140, -1)) - # Output file box - self.out_box = ct.InputCtrl(self, label='Project folder: ', - label_size=(140, -1), - label_style='bold', - value=os.path.abspath(os.curdir), - buttons=True) - # Input file box - self.inp_box = FileListCtrl(self) - - # Options - opt_box = wx.FlexGridSizer(2, 3, 10, 10) - self.opt_chk_useref = wx.CheckBox(self, label='Use reference in refinement') - self.opt_chk_useref.Disable() - self.opt_spc_nres = ct.SpinCtrl(self, - label='No. of Residues: ', - label_size=(160, -1), - ctrl_size=(100, -1), - ctrl_value=500, - ctrl_min=10, - ctrl_max=100000) - procs = multiprocessing.cpu_count() - self.opt_spc_nproc = ct.SpinCtrl(self, - label='No. of Processors: ', - label_size=(160, -1), - ctrl_max=procs, - ctrl_min=1, - ctrl_value=str(int(procs / 2))) - self.opt_btn = wx.Button(self, label='Advanced Options...') - opt_box.AddMany([(self.opt_chk_useref), (0, 0), - (self.opt_spc_nres), - (self.opt_btn), (0, 0), - (self.opt_spc_nproc)]) - opt_box.AddGrowableCol(1) - - # Add to sizers - self.main_sizer.Add(self.project_title, flag=f.expand, border=10) - self.main_sizer.Add(self.out_box, flag=f.expand, border=10) - self.main_sizer.Add(self.inp_box, 1, flag=wx.EXPAND | wx.ALL, border=10) - self.main_sizer.Add(wx.StaticLine(self), flag=wx.EXPAND) - self.main_sizer.Add(opt_box, flag=wx.ALL | wx.EXPAND, border=10) - - # Button bindings - self.out_box.btn_browse.Bind(wx.EVT_BUTTON, self.onOutputBrowse) - self.opt_btn.Bind(wx.EVT_BUTTON, self.onAdvancedOptions) - - def onInputBrowse(self, e): - """ On clincking the Browse button: show the DirDialog and populate 'Input' - box w/ selection """ - inp_dlg = wx.DirDialog(self, "Choose the input directory:", - style=wx.DD_DEFAULT_STYLE) - if inp_dlg.ShowModal() == wx.ID_OK: - self.inp_box.ctr.SetValue(inp_dlg.GetPath()) - inp_dlg.Destroy() - self.update_settings() - e.Skip() - - def update_settings(self): - idxs = self.inp_box.ctr.GetItemCount() - items = [self.inp_box.ctr.GetItemData(i) for i in range(idxs)] - inputs = [] - reference = None - sequence = None - - for i in items: - inp_type = i.type.type.GetString(i.type_selection) - if inp_type in ('processed pickle list', - 'processed pickle folder', - 'processed pickle'): - inputs.append(i.path) - elif inp_type == 'reference MTZ': - reference = i.path - elif inp_type == 'sequence': - sequence = i.path - - self.pparams.data = inputs - - if reference is not None: - self.pparams.hklisoin = reference - if self.opt_chk_useref.GetValue(): - self.pparams.hklrefin = reference - - self.out_dir = self.out_box.ctr.GetValue() - self.pparams.run_no = utils.set_base_dir(out_dir=self.out_dir) # Need to change - self.pparams.title = self.project_title.ctr.GetValue() - self.pparams.n_residues = self.opt_spc_nres.ctr.GetValue() - self.pparams.n_processors = self.opt_spc_nproc.ctr.GetValue() - - update_phil = master_phil.format(python_object=self.pparams) - self.regenerate_params(update_phil) - - def onOutputBrowse(self, e): - """ On clicking the Browse button: show the DirDialog and populate 'Output' - box w/ selection """ - save_dlg = wx.DirDialog(self, "Choose the output directory:", - style=wx.DD_DEFAULT_STYLE) - if save_dlg.ShowModal() == wx.ID_OK: - self.out_box.ctr.SetValue(save_dlg.GetPath()) - save_dlg.Destroy() - self.update_settings() - e.Skip() - - def onAdvancedOptions(self, e): - e.Skip() - - def regenerate_params(self, phil=None): - if phil is not None: - self.prime_phil = master_phil.fetch(source=phil) - else: - self.prime_phil = master_phil - - # Generate Python object and text of parameters - self.generate_phil_string(self.prime_phil) - self.pparams = self.prime_phil.extract() - - def generate_phil_string(self, current_phil): - with utils.Capturing() as txt_output: - current_phil.show() - self.phil_string = '' - for one_output in txt_output: - self.phil_string += one_output + '\n' - - -class InputWindow(pct.PHILDialogPanel): - """ This window will enable all input handling """ - def __init__(self, parent, scope, *args, **kwargs): - super(InputWindow, self).__init__(parent, scope, *args, **kwargs) - - -class PRIMEWindow(IOTABaseFrame): - """ Main launch window for PRIME """ - - def __init__(self, parent, id, title, prefix='prime'): - IOTABaseFrame.__init__(self, parent, id, title, size=(800, 500)) - - self.parent = parent - self.prime_filename = '{}.phil'.format(prefix) - - # Toolbar - self.initialize_toolbar() - self.tb_btn_quit = self.add_tool(label='Quit', - bitmap=('actions', 'exit'), - shortHelp='Exit PRIME') - - self.tb_btn_prefs = self.add_tool(label='Preferences', - bitmap=('apps', 'advancedsettings'), - shortHelp='PRIME GUI Settings') - self.add_toolbar_separator() - self.tb_btn_load = self.add_tool(label='Load Script', - bitmap=('actions', 'open'), - shortHelp='Load PRIME Script') - self.tb_btn_save = self.add_tool(label='Save Script', - bitmap=('actions', 'save'), - shortHelp='Save PRIME Script') - self.tb_btn_reset = self.add_tool(label='Reset', - bitmap=('actions', 'reload'), - shortHelp='Reset Settings') - self.toolbar.AddSeparator() - self.tb_btn_analysis = self.add_tool(label='Recover', - bitmap=('actions', 'list'), - shortHelp='Recover PRIME run') - self.realize_toolbar() - - # Toolbar button bindings - self.Bind(wx.EVT_TOOL, self.onQuit, self.tb_btn_quit) - self.Bind(wx.EVT_TOOL, self.onPreferences, self.tb_btn_prefs) - self.Bind(wx.EVT_TOOL, self.onRecovery, self.tb_btn_analysis) - self.Bind(wx.EVT_TOOL, self.onLoadScript, self.tb_btn_load) - self.Bind(wx.EVT_TOOL, self.onSaveScript, self.tb_btn_save) - self.Bind(wx.EVT_TOOL, self.onReset, self.tb_btn_reset) - - # Add panels (make separate FlexGridSizer to make sure the sizing works) - self.panel_sizer = wx.FlexGridSizer(2, 1, 0, 0) - self.panel_sizer.AddGrowableRow(0) - self.panel_sizer.AddGrowableCol(0) - - # Initialize PHIL index - self.initialize_PRIME_index() - - # Define scopes - self.main_window_scopes = ['title', 'run_no', 'data'] - self.pref_scopes = ['n_processors', 'queue'] - self.pr_scopes = ['scale', 'merge', 'postref'] - self.opt_scopes = ['target_space_group', 'target_unit_cell', - 'target_anomalous_flag', 'n_residues', 'pixel_size_mm'] - - # Instantiate input window - input_scope = self.prime_index.get_scopes(include=self.main_window_scopes) - self.input_window = InputWindow( - self, - size=(600, -1), - scope=input_scope, - phil_index=self.prime_index, - path_extras={ - "file_types": ['processed pickle', - 'reference MTZ', - 'sequence', - 'processed pickle list'], - "folder_types": ['processed pickle folder'], - "data_types": ['processed pickle'], - "input_filter": "processed" - }) - - # Front options panel - self.bottom_sizer = wx.GridBagSizer(5, 5) - line = wx.StaticLine(self, style=wx.LI_HORIZONTAL, size=(-1, -1)) - self.bottom_sizer.Add(line, pos=(0, 0), span=(1, 3), - flag=wx.EXPAND | wx.TOP | wx.BOTTOM, border=5) - - opt_scope = self.prime_index.get_scopes(include=self.opt_scopes) - self.option_panel = pct.PHILDialogPanel(self, scope=opt_scope, - phil_index=self.prime_index) - self.bottom_sizer.Add(self.option_panel, pos=(1, 0), span=(2, 1), - flag=wx.EXPAND) - line = wx.StaticLine(self, style=wx.LI_VERTICAL, size=(-1, -1)) - self.bottom_sizer.Add(line, pos=(1, 1), span=(2, 1), - flag=wx.EXPAND | wx.RIGHT | wx.LEFT, border=15) - self.bottom_sizer.AddGrowableCol(0) - self.bottom_sizer.AddGrowableRow(0) - - # Front options - btn_box = wx.StaticBox(self, label='') - btn_sizer = wx.StaticBoxSizer(btn_box, wx.VERTICAL) - self.opt_btn_prm = wx.Button(self, label='Post-refinement...') - self.opt_btn_adv = wx.Button(self, label='Advanced Options...') - btn_sizer.Add(self.opt_btn_prm, flag=wx.EXPAND | wx.ALL, border=5) - btn_sizer.Add(self.opt_btn_adv, flag=wx.EXPAND | wx.ALL, border=5) - self.bottom_sizer.Add(btn_sizer, pos=(1, 2), flag=wx.EXPAND) - - run_bmp = bitmaps.fetch_icon_bitmap('actions', 'run') - self.btn_run = ct.GradButton(self, - size=(180, 40), - start_color=(161, 217, 155), - highlight_color=(199, 233, 192), - gradient_percent=-25, - bmp=run_bmp, - label=' RUN PRIME', - label_size=big_button_font_size) - self.bottom_sizer.Add(self.btn_run, pos=(2, 2), - flag=wx.SHAPED | wx.ALIGN_BOTTOM) - self.btn_run.Disable() - - # Option button bindings - self.Bind(wx.EVT_BUTTON, self.onPostRefOptions, self.opt_btn_prm) - self.Bind(wx.EVT_BUTTON, self.onAdvancedOptions, self.opt_btn_adv) - self.Bind(wx.EVT_BUTTON, self.onRun, self.btn_run) - - # Add panels to sizers and lay out the window - self.panel_sizer.Add(self.input_window, 1, flag=wx.EXPAND) - self.panel_sizer.Add(self.bottom_sizer, flag=wx.EXPAND) - self.main_sizer.Add(self.panel_sizer, 1, flag=wx.EXPAND | wx.ALL, border=15) - self.Fit() - self.Layout() - - def OnAboutBox(self, e): - ''' About dialog ''' - info = wx.AboutDialogInfo() - info.SetName('PRIME') - info.SetWebSite('http://cci.lbl.gov/xfel') - info.SetLicense(prime_license) - info.SetDescription(prime_description) - info.AddDeveloper('Monarin Uervirojnangkoorn') - info.AddDeveloper('Axel Brunger') - wx.AboutBox(info) - - def open_options_dialog(self, phil_index, include=None, name=None, - exclude=None): - if not name: - if isinstance(include, list): - name = include[0] - else: - name = include - if not name or name.isspace(): - name = 'options' - - phil_scope = phil_index.get_scopes(include=include, - exclude=exclude) - phil_dlg = pct.PHILDialog(self, - scope=phil_scope, - phil_index=phil_index, - name=name) - - if phil_dlg.ShowModal() == wx.ID_OK: - OK = True - else: - OK = False - - phil_dlg.Destroy() - return OK - - def onPreferences(self, e): - self.open_options_dialog(phil_index=self.prime_index, - include=self.pref_scopes) - - def onPostRefOptions(self, e): - self.open_options_dialog(phil_index=self.prime_index, - include=self.pr_scopes) - - def onAdvancedOptions(self, e): - not_adv = (self.main_window_scopes + - self.opt_scopes + - self.pref_scopes + - self.pr_scopes) - self.open_options_dialog(phil_index=self.prime_index, - exclude=not_adv) - - def onInput(self, e): - if self.input_window.inp_box.ctr.GetValue() != '': - self.set_tool_state(self.tb_btn_run, enable=True) - else: - self.set_tool_state(self.tb_btn_run, enable=False) - - def initialize_PRIME_index(self): - self.prime_index = make_phil_index(master_phil=master_phil) - self.pparams = self.prime_index.get_python_object(make_copy=True) - self.out_dir = os.path.dirname(self.pparams.run_no) if \ - self.pparams.run_no else os.curdir - self.update_PRIME_index() - - def update_PRIME_index(self, phil=None, update_run_no=True): - # update from existing PHIL - if phil: - if not isinstance(phil, str): - try: - phil = phil.as_str() - except Exception as e: - raise Sorry('PRIME GUI ERROR: Cannot read PHIL object! ', e) - self.prime_index.update_phil(phil_string=phil) - self.pparams = self.prime_index.get_python_object(make_copy=True) - - # update n_processors - if self.pparams.n_processors <= 1: - self.pparams.n_processors = int(multiprocessing.cpu_count() * 0.5) - - # update output folder - if update_run_no: - if self.pparams.run_no is None: - self.pparams.run_no = utils.set_base_dir() - else: - self.out_dir = os.path.dirname(self.pparams.run_no) - self.pparams.run_no = utils.set_base_dir(out_dir=self.out_dir) - - # Update index and PHIL from params - self.prime_index.update_from_python(python_object=self.pparams) - - - def reset_PRIME_index(self): - self.prime_index.reset_phil(reindex=True) - - def sanity_check(self): - ''' - Goes through and checks that the key parameters are populated; pops - up an error message if they are not - :return: True if satisfied, False if not - ''' - - errors = self.option_panel.collect_errors() - if errors: - wx.MessageBox(caption='Errors in Settings!', - message='Correct errors in the following settings: {}' - ''.format('\n'.join(errors.keys())), - style=wx.OK|wx.ICON_EXCLAMATION) - - return True - - def onRecovery(self, e): - - # Change current directory to the parent directory of run_no (which may - # be changed by user) - run_no = self.input_window.get_value('run_no') - curdir = os.path.dirname(run_no) - if curdir: - if curdir.endswith('/prime'): - curdir = os.path.dirname(curdir) - os.chdir(curdir) - - # Find finished runs and display results - p_folder = os.path.abspath('{}/prime'.format(os.curdir)) - - if not os.path.isdir(p_folder): - open_dlg = wx.DirDialog(self, "Choose the integration run:", - style=wx.DD_DEFAULT_STYLE| - wx.DD_CHANGE_DIR) - if open_dlg.ShowModal() == wx.ID_OK: - p_folder = open_dlg.GetPath() - open_dlg.Destroy() - else: - open_dlg.Destroy() - return - - paths = [os.path.join(p_folder, p) for p in os.listdir(p_folder)] - paths = [p for p in paths if (os.path.isdir(p) and - os.path.basename(p).isdigit())] - - path_dlg = dlg.RecoveryDialog(self) - path_dlg.insert_paths(paths) - - if path_dlg.ShowModal() == wx.ID_OK: - selected = path_dlg.selected - recovery = path_dlg.recovery_mode - prime_path = selected[1] - prime_status = selected[0] - settings_file = os.path.join(prime_path, 'settings.phil') - - # If neither log-file nor stat file are found, terminate; otherwise - # import settings - if not os.path.isfile(settings_file): - wx.MessageBox('Cannot Import This Run \n(No Files Found!)', - 'Info', wx.OK | wx.ICON_ERROR) - return - else: - with open(settings_file, 'r') as sf: - phil_string = sf.read() - read_phil = ip.parse(phil_string) - - self.update_PRIME_index(phil=read_phil, update_run_no=False) - self.update_input_window() - - # If any cycles (or full run) were completed, show results - if prime_status == 'Unknown': - return - - if recovery == 0: - self.prime_run_window = PRIMERunWindow(self, -1, - title='PRIME Output', - params=self.pparams, - prime_file=settings_file, - recover=True) - self.prime_run_window.place_and_size(set_by='parent') - self.prime_run_window.Show(True) - self.prime_run_window.recover() - - def init_settings(self): - # Automatically advance run_no - old_run_no = self.input_window.get_value('run_no') - if os.path.isdir(old_run_no): - new_run_no = utils.set_base_dir(dirname=os.path.dirname(old_run_no)) - self.input_window.change_value('run_no', new_run_no) - - input_phil_string = self.input_window.GetPHIL(expand=True) - sel_phil_string = self.option_panel.GetPHIL(expand=True) - phil_string = input_phil_string + '\n' + sel_phil_string - - self.prime_index.update_phil(phil_string=phil_string) - self.pparams = self.prime_index.get_python_object(make_copy=True) - self.prime_phil = self.prime_index.working_phil - - return self.sanity_check() - - def onRun(self, e): - # Run full processing - - if self.init_settings(): - source_dir = os.path.dirname(self.pparams.run_no) - if not os.path.isdir(source_dir): - try: - os.makedirs(source_dir) - except IOError as e: - raise Sorry('PRIME ERROR: Could not create folder {}: {}' - ''.format(source_dir, e)) - prime_file = os.path.join(source_dir, self.prime_filename) - with open(prime_file, 'w') as pf: - pf.write(self.prime_phil.as_str()) - - self.prime_run_window = PRIMERunWindow(self, -1, - title='PRIME Output', - params=self.pparams, - prime_file=prime_file) - self.prime_run_window.prev_pids = easy_run.fully_buffered('pgrep -u {} {}' - ''.format(user, python)).stdout_lines - self.prime_run_window.place_and_size(set_by='parent') - self.prime_run_window.Show(True) - - def onSequence(self, e): - pass - - def onSaveScript(self, e): - self.init_settings() - - # Generate text of params - final_phil = master_phil.format(python_object=self.pparams) - with utils.Capturing() as txt_output: - final_phil.show() - txt_out = '' - for one_output in txt_output: - txt_out += one_output + '\n' - - # Save param file - save_dlg = wx.FileDialog(self, - message="Save PRIME Script", - defaultDir=os.curdir, - defaultFile="*.phil", - wildcard="*.phil", - style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT - ) - if save_dlg.ShowModal() == wx.ID_OK: - with open(save_dlg.GetPath(), 'w') as savefile: - savefile.write(txt_out) - - save_dlg.Destroy() - - def onLoadScript(self, e): - # Extract params from file - load_dlg = wx.FileDialog(self, - message="Load script file", - defaultDir=os.curdir, - defaultFile="*.phil", - wildcard="*.phil", - style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST, - ) - if load_dlg.ShowModal() == wx.ID_OK: - script = load_dlg.GetPaths()[0] - out_dir = os.path.dirname(script) - self.prime_filename=os.path.basename(script) - self.load_script(out_dir=out_dir) - load_dlg.Destroy() - - def onReset(self, e): - self.reset_settings() - - def load_script(self, out_dir): - ''' Loads PRIME script ''' - script = os.path.join(out_dir, self.prime_filename) - with open(script, 'r') as sf: - phil_string = sf.read() - - self.update_PRIME_index(phil=phil_string) - self.update_input_window() - - def update_input_window(self, reset=False): - ''' Update input window with current (or default) params''' - self.input_window.redraw_panel(reset=reset) - self.option_panel.redraw_panel(reset=reset) - - def reset_settings(self): - self.prime_phil = master_phil - self.initialize_PRIME_index() - self.update_input_window(reset=True) - - def onQuit(self, e): - - # Check if proc_thread exists - try: - if hasattr(self, 'prime_run_window'): - if hasattr(self.prime_run_window, 'prime_process'): - - # Check if proc_thread is running - if self.prime_run_window.prime_process.is_alive(): - self.prime_run_window.abort_run(verbose=False) - - # Close window only when thread is dead - while self.prime_run_window.prime_process.is_alive(): - print ('THREAD ALIVE? ', - self.prime_run_window.prime_process.is_alive()) - continue - except Exception: - pass - - self.Close() - -# ---------------------------- Processing Window --------------------------- # - - -class LogTab(IOTABasePanel): - def __init__(self, parent, *args, **kwargs): - super(LogTab, self).__init__(parent=parent, *args, **kwargs) - - self.log_sizer = wx.BoxSizer(wx.VERTICAL) - self.log_window = wx.TextCtrl(self, - style=wx.TE_MULTILINE | wx.TE_READONLY | wx.TE_DONTWRAP) - self.log_window.SetFont(wx.Font(9, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False)) - self.log_sizer.Add(self.log_window, proportion=1, flag= wx.EXPAND | wx.ALL, border=10) - self.SetSizer(self.log_sizer) - - -class RuntimeTab(IOTABasePanel): - def __init__(self, parent, params=None, *args, **kwargs): - super(RuntimeTab, self).__init__(parent=parent, *args, **kwargs) - self.pparams = params - - # For some reason MatPlotLib 2.2.3 on GTK 6 does not create transparent - # patches (tried set_visible(False), set_facecolor("none"), set_alpha(0), - # to no avail). Thus, for GTK only, setting facecolor to background - # color; otherwide to 'none'. This may change if other tests reveal the - # same problem in other systems. - if wx.Platform == '__WXGTK__': - bg_color = [i/255 for i in self.GetBackgroundColour()] - else: - bg_color = 'none' - - mpl.rc('font', family='sans-serif', size=10) - mpl.rc('mathtext', default='regular') - - # Create figure - self.prime_sizer = wx.BoxSizer(wx.VERTICAL) - self.prime_figure = Figure() - # self.prime_figure.patch.set_alpha(0) - self.prime_figure.patch.set_facecolor(color=bg_color) - - # Create nested GridSpec - gsp = gridspec.GridSpec(2, 2, height_ratios=[2, 3]) - self.cc_axes = self.prime_figure.add_subplot(gsp[0]) - self.comp_axes = self.prime_figure.add_subplot(gsp[1]) - self.mult_axes = self.comp_axes.twinx() - self.rej_table = self.prime_figure.add_subplot(gsp[2]) - - gsub = gridspec.GridSpecFromSubplotSpec(3, 1, - subplot_spec=gsp[3], - hspace=0) - self.bcc_axes = self.prime_figure.add_subplot(gsub[0]) - self.bcomp_axes = self.prime_figure.add_subplot(gsub[1], - sharex=self.bcc_axes) - self.bmult_axes = self.prime_figure.add_subplot(gsub[2], - sharex=self.bcc_axes) - self.draw_axes() - - # Set layout, create canvas, add to sizer - self.prime_figure.set_tight_layout(True) - self.canvas = FigureCanvas(self, -1, self.prime_figure) - self.prime_sizer.Add(self.canvas, proportion=1, flag =wx.EXPAND) - - # Initialize main sizer - self.SetSizer(self.prime_sizer) - - def draw_axes(self): - self.rej_table.axis('off') - self.cc_axes.set_title(r'$CC_{1/2}$', fontsize=12) - self.cc_axes.set_xlabel('Cycle') - self.cc_axes.set_ylabel(r'$CC_{1/2}$ (%)') - self.cc_axes.ticklabel_format(axis='y', style='plain') - self.cc_axes.set_xticks([tick for tick in self.cc_axes.get_xticks() - if tick % 1 == 0]) - - self.comp_axes.set_title('Completeness / Multiplicity', fontsize=12) - self.comp_axes.set_xlabel('Cycle') - self.comp_axes.set_ylabel('Completeness (%)') - self.mult_axes.set_ylabel('# of Observations') - self.comp_axes.ticklabel_format(axis='y', style='plain') - self.mult_axes.ticklabel_format(axis='y', style='plain') - self.comp_axes.set_xticks([tick for tick in self.comp_axes.get_xticks() - if tick % 1 == 0]) - - self.bcc_axes.yaxis.get_major_ticks()[0].label1.set_visible(False) - self.bcc_axes.yaxis.get_major_ticks()[-1].label1.set_visible(False) - - if self.pparams.target_anomalous_flag: - self.bcc_axes.set_ylabel(r'$CC_{1/2}$ anom (%)') - else: - self.bcc_axes.set_ylabel(r'$CC_{1/2}$ (%)') - # plt.setp(self.bcc_axes.get_xticklabels(), visible=False) - self.bcc_axes.set_xticklabels([]) - self.bcomp_axes.yaxis.get_major_ticks()[0].label1.set_visible(False) - self.bcomp_axes.yaxis.get_major_ticks()[-1].label1.set_visible(False) - self.bcomp_axes.set_ylabel("Comp (%)") - # plt.setp(self.bcomp_axes.get_xticklabels(), visible=False) - self.bcomp_axes.set_xticklabels([]) - self.bmult_axes.yaxis.get_major_ticks()[0].label1.set_visible(False) - self.bmult_axes.yaxis.get_major_ticks()[-1].label1.set_visible(False) - self.bmult_axes.set_xlabel(r"Resolution ($\AA$)") - self.bmult_axes.set_ylabel("# of Obs") - - def draw_plots(self, info, total_cycles): - - # Plot mean CC1/2 - meanCC = info['total_cc12'] - cycles = list(range(len(meanCC))) - self.cc_axes.clear() - self.cc_axes.set_xlim(0, total_cycles) - self.cc_axes.set_ylim(0, 100) - self.cc_axes.plot(cycles, meanCC, 'o', c='#2b8cbe', ls='-', lw=3) - - # Plot mean completeness and multiplicity - mean_comp = info['total_completeness'] - mean_mult = info['total_n_obs'] - cycles = list(range(len(mean_comp))) - self.comp_axes.clear() - self.mult_axes.clear() - self.comp_axes.set_xlim(0, total_cycles) - self.comp_axes.set_ylim(0, 100) - self.mult_axes.set_xlim(0, total_cycles) - self.comp_axes.plot(cycles, mean_comp, c='#f03b20', ls='-', lw=2) - comp = self.comp_axes.scatter(cycles, mean_comp, marker='o', s=25, - edgecolors='black', color='#f03b20') - self.mult_axes.plot(cycles, mean_mult, c='#feb24c', ls='-', lw=2) - mult = self.mult_axes.scatter(cycles, mean_mult, marker='o', s=25, - edgecolors='black', color='#feb24c') - labels = ['Completeness', 'Multiplicity'] - self.comp_axes.legend([comp, mult], labels, loc='upper right', - fontsize=9, fancybox=True) - - # Binned bar plots - x = info['binned_resolution'][-1] - bins = np.arange(len(x)) - xlabels = ["{:.2f}".format(i) for i in x] - sel_bins = bins[0::len(bins) // 6] - sel_xlabels = [xlabels[t] for t in sel_bins] - - # plot binned stats - self.bcc_axes.clear() - if self.pparams.target_anomalous_flag: - binned_cc = [c * 100 for c in info['binned_cc12_anom'][-1]] - else: - binned_cc = [c * 100 for c in info['binned_cc12'][-1]] - - self.bcc_axes.bar(bins, binned_cc, color='#2b8cbe', - alpha=0.5, width=1, lw=0) - self.bcc_axes.step(bins, binned_cc, color='blue', where='mid') - self.bcomp_axes.clear() - self.bcomp_axes.bar(bins, info['binned_completeness'][-1], - alpha=0.5, color='#f03b20', width=1, lw=0) - self.bcomp_axes.step(bins, info['binned_completeness'][-1], color='red', - where='mid') - self.bmult_axes.clear() - self.bmult_axes.bar(bins, info['binned_n_obs'][-1], - alpha=0.5, color='#feb24c', width=1, lw=0) - self.bmult_axes.step(bins, info['binned_n_obs'][-1], color='orange', - where='mid') - - # Set x-axis tick labels - self.bmult_axes.set_xticks(sel_bins) - self.bmult_axes.set_xticklabels(sel_xlabels) - self.draw_axes() - - # Rejection table - txt = 'No. good frames: {}\n' \ - 'No. bad CC frames: {}\n' \ - 'No. bad G frames: {}\n' \ - 'No. bad unit cell frames: {}\n' \ - 'No. bad gamma_e frames: {}\n' \ - 'No. bad SE frames: {}\n' \ - 'No. observations: {}\n' \ - ''.format(info['n_frames_good'][-1], - info['n_frames_bad_cc'][-1], - info['n_frames_bad_G'][-1], - info['n_frames_bad_uc'][-1], - info['n_frames_bad_gamma_e'][-1], - info['n_frames_bad_SE'][-1], - info['n_observations'][-1]) - - self.rej_table.clear() - self.rej_table.axis('off') - font = {'family': 'monospace', - 'color': 'darkblue', - 'weight': 'normal', - 'size': 11, - 'linespacing': 2.5 - } - self.rej_table.text(0, 0.85, txt, fontdict=font, - transform=self.rej_table.transAxes, - va='top') - - # Redraw canvas - self.canvas.draw_idle() - self.canvas.Refresh() - -class SummaryTab(IOTABaseScrolledPanel): - def __init__(self, - parent, - pparams, - info, - *args, **kwargs): - super(SummaryTab, self).__init__(parent=parent, *args, **kwargs) - - self.info = info - self.pparams = pparams - - self.summary_sizer = wx.BoxSizer(wx.VERTICAL) - - sfont = wx.Font(norm_font_size, wx.FONTFAMILY_DEFAULT, - wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) - bfont = wx.Font(norm_font_size, wx.FONTFAMILY_DEFAULT, - wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_BOLD) - self.SetFont(bfont) - - # Run information - run_box = wx.StaticBox(self, label='Run Information') - run_box.SetFont(sfont) - run_box_sizer = wx.StaticBoxSizer(run_box, wx.VERTICAL) - run_box_grid = wx.FlexGridSizer(3, 2, 5, 20) - self.title_txt = wx.StaticText(self, label='') - self.title_txt.SetFont(sfont) - self.folder_txt = wx.StaticText(self, label='') - self.folder_txt.SetFont(sfont) - - run_box_grid.AddMany([(wx.StaticText(self, label='Title')), - (self.title_txt, 1, wx.EXPAND), - (wx.StaticText(self, label='Directory')), - (self.folder_txt, 1, wx.EXPAND)]) - - run_box_grid.AddGrowableCol(1, 1) - run_box_sizer.Add(run_box_grid, flag=wx.EXPAND | wx.ALL, border=10) - self.summary_sizer.Add(run_box_sizer, flag=wx.EXPAND | wx.ALL, border=10) - - # Merging statistics box - tb1_box = wx.StaticBox(self, label='Merging Statistics') - tb1_box.SetFont(sfont) - tb1_box_sizer = wx.StaticBoxSizer(tb1_box, wx.VERTICAL) - self.tb1_box_grid = wx.GridBagSizer(5, 20) - - # Table 1 - self.tb1_table = Plotter(self, info=self.info) - self.tb1_table.initialize_figure(figsize=(0.1, 0.1)) - self.tb1_box_grid.Add(self.tb1_table, pos=(0, 0), span=(3, 1), - flag=wx.EXPAND) - self.tb1_box_grid.AddGrowableCol(0) - self.tb1_box_grid.AddGrowableRow(2) - - # Buttons - line_bmp = bitmaps.fetch_custom_icon_bitmap('line_graph24') - self.btn_stats = ct.GradButton(self, - bmp=line_bmp, - label=' Statistical charts') - txt_bmp = bitmaps.fetch_icon_bitmap('mimetypes', 'txt', scale=(24, 24)) - self.btn_table1 = ct.GradButton(self, - bmp=txt_bmp, - label=' Output Table 1') - self.tb1_box_grid.Add(self.btn_stats, pos=(0, 1), flag=wx.ALIGN_RIGHT) - self.tb1_box_grid.Add(self.btn_table1, pos=(1, 1), flag=wx.ALIGN_RIGHT) - - # Insert into sizers - tb1_box_sizer.Add(self.tb1_box_grid, 1, flag=wx.EXPAND | wx.ALL, border=10) - self.summary_sizer.Add(tb1_box_sizer, 1, flag=wx.EXPAND | wx.ALL, border=10) - self.summary_sizer.Add((10, -1), flag=wx.EXPAND) - - # Button bindings - self.Bind(wx.EVT_BUTTON, self.onPlotStats, self.btn_stats) - self.Bind(wx.EVT_BUTTON, self.onWriteTableOne, self.btn_table1) - - self.SetSizer(self.summary_sizer) - self.Refresh() - self.Layout() - self.SetupScrolling() - - - def update(self): - # Table 1 - self.tb1_table.table_one_figure() - self.tb1_table.Layout() - self.SetupScrolling() - - def initialize_standalone_plot(self, figsize=(8, 8)): - self.plot_window = PlotWindow(self, -1, title='PRIME Statistics') - self.plot = Plotter(self.plot_window, info=self.info, - anomalous_flag=self.pparams.target_anomalous_flag) - self.plot_window.plot_panel = self.plot - self.plot.initialize_figure(figsize=figsize) - - def show_plot(self): - self.plot_window.add_plot_to_window() - self.plot_window.place_and_size(set_by='parent', set_size=True, - position=(25, 25)) - self.plot_window.Show() - - def onPlotStats(self, e): - self.initialize_standalone_plot() - self.plot.stat_charts() - self.show_plot() - - def onWriteTableOne(self, e): - ''' Write Table 1 to a file ''' - - save_dlg = wx.FileDialog(self, - message="Save Table 1", - defaultDir=os.curdir, - defaultFile="table_1.txt", - wildcard="*.txt", - style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT - ) - if save_dlg.ShowModal() == wx.ID_OK: - with open(save_dlg.GetPath(), 'a') as tb1_file: - tb1_text = self.tb1_table.table_one_text() - tb1_file.write(tb1_text) - - -class PRIMERunWindow(IOTABaseFrame): - ''' New frame that will show processing info ''' - - def __init__(self, parent, id, title, params, - prime_file, mp_method='python', command=None, recover=False): - IOTABaseFrame.__init__(self, parent, id, title, size=(800, 900), - style=wx.SYSTEM_MENU | - wx.CAPTION | - wx.CLOSE_BOX | - wx.RESIZE_BORDER) - - self.logtext = '' - self.pparams = params - self.prime_file = prime_file - self.out_file = os.path.join(self.pparams.run_no, 'log.txt') - self.bookmark = 0 - self.prev_pids = [] - self.aborted = False - self.command=command - self.mp_method = mp_method - self.current_cycle = -1 - - # Toolbar - self.initialize_toolbar() - self.tb_btn_abort = self.add_tool(label='Abort', - bitmap=('actions', 'stop'), - shortHelp='Abort') - self.realize_toolbar() - - # Status box - self.status_panel = wx.Panel(self) - self.status_sizer = wx.BoxSizer(wx.VERTICAL) - self.status_box = wx.StaticBox(self.status_panel, label='Status') - self.status_box_sizer = wx.StaticBoxSizer(self.status_box, wx.HORIZONTAL) - self.status_txt = wx.StaticText(self.status_panel, label='') - self.status_box_sizer.Add(self.status_txt, flag=wx.ALL | wx.ALIGN_CENTER, - border=10) - self.status_sizer.Add(self.status_box_sizer, - flag=wx.EXPAND | wx.ALL, border=3) - self.status_panel.SetSizer(self.status_sizer) - - # Tabbed output window(s) - self.prime_panel = wx.Panel(self) - self.prime_nb = wx.Notebook(self.prime_panel, style=0) - self.log_tab = LogTab(self.prime_nb) - self.graph_tab = RuntimeTab(self.prime_nb, params=self.pparams) - self.prime_nb.AddPage(self.log_tab, 'Log') - self.prime_nb.AddPage(self.graph_tab, 'Charts') - self.prime_nb.SetSelection(1) - self.prime_sizer = wx.BoxSizer(wx.VERTICAL) - self.prime_sizer.Add(self.prime_nb, 1, flag=wx.EXPAND | wx.ALL, border=3) - self.prime_panel.SetSizer(self.prime_sizer) - - self.main_sizer.Add(self.status_panel, flag=wx.EXPAND | wx.ALL, border=3) - self.main_sizer.Add(self.prime_panel, 1, flag=wx.EXPAND | wx.ALL, border=3) - - #Processing status bar - self.sb.SetFieldsCount(2) - self.sb.SetStatusWidths([-1, -2]) - - # Output gauge in status bar - self.gauge_prime = wx.Gauge(self.sb, -1, - style=wx.GA_HORIZONTAL | wx.GA_SMOOTH) - rect = self.sb.GetFieldRect(0) - self.gauge_prime.SetPosition((rect.x + 2, rect.y + 2)) - self.gauge_prime.SetSize((rect.width - 4, rect.height - 4)) - self.gauge_prime.Hide() - - # Output polling timer - self.timer = wx.Timer(self) - - # Event bindings - self.Bind(thr.EVT_ALLDONE, self.onFinishedProcess) - self.sb.Bind(wx.EVT_SIZE, self.onStatusBarResize) - self.Bind(wx.EVT_TIMER, self.onTimer, id=self.timer.GetId()) - - # Button bindings - self.Bind(wx.EVT_TOOL, self.onAbort, self.tb_btn_abort) - - if not recover: - self.run() - - def onStatusBarResize(self, e): - rect = self.sb.GetFieldRect(0) - self.gauge_prime.SetPosition((rect.x + 2, rect.y + 2)) - self.gauge_prime.SetSize((rect.width - 4, rect.height - 4)) - - def onAbort(self, e): - self.status_txt.SetForegroundColour('red') - self.status_txt.SetLabel('Aborting...') - self.toolbar.EnableTool(self.tb_btn_abort.GetId(), False) - self.abort_run() - - def abort_run(self, verbose=True): - if self.mp_method == 'python': - self.pids = easy_run.fully_buffered('pgrep -u {} {}' - ''.format(user, python)).stdout_lines - self.pids = [i for i in self.pids if i not in self.prev_pids] - for i in self.pids: - easy_run.fully_buffered('kill -9 {}'.format(i)) - if verbose: - print ('killing PID {}'.format(i)) - self.aborted = True - - def run(self): - self.status_txt.SetForegroundColour('black') - self.status_txt.SetLabel('Running...') - self.gauge_prime.SetRange(self.pparams.n_postref_cycle) - - self.prime_process = thr.PRIMEThread(self, - self.prime_file, - self.out_file, - command=self.command, - verbose=True, - debug=True) - self.prime_process.start() - self.timer.Start(5000) - - def recover(self): - self.status_txt.SetForegroundColour('black') - self.status_txt.SetLabel('Displaying results from {}' - ''.format(self.pparams.run_no)) - - # Plot results - self.plot_runtime_results() - self.plot_final_results() - - def display_log(self): - ''' Display PRIME stdout ''' - if os.path.isfile(self.out_file): - with open(self.out_file, 'r') as out: - out.seek(self.bookmark) - output = out.readlines() - self.bookmark = out.tell() - - ins_pt = self.log_tab.log_window.GetInsertionPoint() - for i in output: - self.log_tab.log_window.AppendText(i) - self.log_tab.log_window.SetInsertionPoint(ins_pt) - - - def plot_runtime_results(self): - ''' Plot results for each cycle upon cycle completion ''' - - # Find pickle_*.stat file - stats_folder = os.path.join(self.pparams.run_no, 'stats') - stat_files = [os.path.join(stats_folder, i) for i in - os.listdir(stats_folder) if i.endswith('stat')] - - if stat_files != []: - assert len(stat_files) == 1 - stat_file = stat_files[0] - if os.path.isfile(stat_file): - info = ep.load(stat_file) - else: - info = {} - else: - info = {} - - if 'binned_resolution' in info: - self.graph_tab.draw_plots(info, self.pparams.n_postref_cycle) - self.current_cycle = len(info['total_cc12']) - 1 - - def plot_final_results(self): - ''' Plot final results ''' - - # Find pickle_*.stat file - stats_folder = os.path.join(self.pparams.run_no, 'stats') - stat_files = [os.path.join(stats_folder, i) for i in - os.listdir(stats_folder) if i.endswith('stat')] - - if stat_files != []: - assert len(stat_files) == 1 - stat_file = stat_files[0] - if os.path.isfile(stat_file): - info = ep.load(stat_file) - else: - info = {} - else: - info = {} - - self.summary_tab = SummaryTab(self.prime_nb, - self.pparams, - info) - - self.summary_tab.title_txt.SetLabel(self.pparams.title) - self.summary_tab.folder_txt.SetLabel(self.pparams.run_no) - self.summary_tab.update() - - # Update log - self.display_log() - - # Display summary - self.prime_nb.AddPage(self.summary_tab, 'Analysis') - self.prime_nb.SetSelection(2) - - def onTimer(self, e): - # If not done yet, write settings file for this run - settings_file = os.path.join(self.pparams.run_no, 'settings.phil') - if not os.path.isfile(settings_file): - try: - with open(self.prime_file, 'r') as pf: - settings = pf.read() - with open(settings_file, 'w') as sf: - sf.write(settings) - except Exception: - pass - - # Inspect output and update gauge - self.gauge_prime.Show() - if self.current_cycle == -1: - self.sb.SetStatusText(('Merging...'), 1) - self.gauge_prime.SetValue(0) - else: - self.sb.SetStatusText('Macrocycle {} of {} completed...' \ - ''.format(self.current_cycle, - self.pparams.n_postref_cycle), 1) - self.gauge_prime.SetValue(self.current_cycle) - - # Plot runtime results - self.plot_runtime_results() - - # Update log - self.display_log() - - # Sense aborted run - if self.aborted: - self.final_step() - - # Sense end of cycle - if self.current_cycle >= self.pparams.n_postref_cycle: - self.final_step() - - - def onFinishedProcess(self, e): - self.final_step() - - - def final_step(self): - font = self.status_txt.GetFont() - font.SetWeight(wx.FONTWEIGHT_BOLD) - - # Check for aborted run - if self.aborted: - self.sb.SetStatusText(('Aborted by user'), 1) - self.status_txt.SetForegroundColour('orange') - self.status_txt.SetLabel('ABORTED BY USER') - self.status_txt.SetFont(font) - else: - self.sb.SetStatusText(('Run finished'), 1) - self.status_txt.SetForegroundColour('blue') - self.status_txt.SetLabel('DONE') - self.status_txt.SetFont(font) - - # Show final results - self.plot_runtime_results() - self.plot_final_results() - - # Copy final results to special folder - import shutil - - # Make the folder - fin_folder = "merged_dataset_{}".format(os.path.basename(self.pparams.run_no)) - dst_dir = os.path.abspath(os.path.join(os.curdir, fin_folder)) - if os.path.isdir(fin_folder): # Just delete it, sheesh - shutil.rmtree(fin_folder) - os.makedirs(dst_dir) - - # Copy files - run_no = int(os.path.basename(self.pparams.run_no)) - src_dir = os.path.abspath(self.pparams.run_no) - src_mtz = 'postref_cycle_{}_merge.mtz'.format(self.current_cycle) - dst_mtz = 'run_{:03d}_final_merged.mtz'.format(run_no) - shutil.copyfile(src=os.path.join(src_dir, src_mtz), - dst=os.path.join(dst_dir, dst_mtz)) - src_hkl = 'postref_cycle_{}_merge.hkl'.format(self.current_cycle) - dst_hkl = 'run_{:03d}_final_merged.hkl'.format(run_no) - shutil.copyfile(src=os.path.join(src_dir, src_hkl), - dst=os.path.join(dst_dir, dst_hkl)) - dst_log = 'run_{}_log.txt'.format(run_no) - shutil.copyfile(src=os.path.join(src_dir, 'log.txt'), - dst=os.path.join(dst_dir, dst_log)) - - # Finish up - self.display_log() - self.gauge_prime.Hide() - self.toolbar.EnableTool(self.tb_btn_abort.GetId(), False) - self.timer.Stop() - - -class FileListCtrl(ct.CustomListCtrl): - ''' File list window for the input tab ''' - - def __init__(self, parent, size=(-1, 300)): - ct.CustomListCtrl.__init__(self, parent=parent, size=size) - - self.main_window = parent.GetParent() - self.input_window = parent - - # Initialize dictionaries for imported data types - self.all_data_images = {} - self.all_img_objects = {} - self.all_proc_pickles = {} - - # Generate columns - self.ctr.InsertColumn(0, "Path") - self.ctr.InsertColumn(1, "Input Type") - self.ctr.InsertColumn(2, "Action") - self.ctr.setResizeColumn(1) - - # Add file / folder buttons - self.button_sizer = wx.BoxSizer(wx.HORIZONTAL) - self.btn_add_file = wx.Button(self, label='Add File...') - self.btn_add_dir = wx.Button(self, label='Add Folder...') - self.button_sizer.Add(self.btn_add_file) - self.button_sizer.Add(self.btn_add_dir, flag=wx.LEFT, border=10) - - self.sizer.Add(self.button_sizer, flag=wx.TOP | wx.BOTTOM, border=10) - - # Event bindings - self.Bind(wx.EVT_BUTTON, self.onAddFile, self.btn_add_file) - self.Bind(wx.EVT_BUTTON, self.onAddFolder, self.btn_add_dir) - - def view_all_images(self): - if self.ctr.GetItemCount() > 0: - file_list = [] - for i in range(self.ctr.GetItemCount()): - type_ctrl = self.ctr.GetItemWindow(i, col=1).type - type_choice = type_ctrl.GetString(type_ctrl.GetSelection()) - if type_choice in ('processed pickle folder'): - for root, dirs, files in os.walk(self.ctr.GetItemText(i)): - for filename in files: - file_list.append(os.path.join(root, filename)) - if type_choice in ('processed pickle'): - file_list.append(self.ctr.GetItemText(i)) - - try: - file_string = ' '.join(file_list) - easy_run.fully_buffered('cctbx.image_viewer {}'.format(file_string)) - except Exception as e: - print (e) - - else: - wx.MessageBox('No data found', 'Error', wx.OK | wx.ICON_ERROR) - - def onAddFile(self, e): - file_dlg = wx.FileDialog(self, - message="Load File", - defaultDir=os.curdir, - defaultFile="*", - wildcard="*", - style=wx.FD_OPEN | - wx.FD_FILE_MUST_EXIST) - if file_dlg.ShowModal() == wx.ID_OK: - self.add_item(file_dlg.GetPaths()[0]) - file_dlg.Destroy() - e.Skip() - - def onAddFolder(self, e): - dlg = wx.DirDialog(self, "Load Folder:", - style=wx.DD_DEFAULT_STYLE) - if dlg.ShowModal() == wx.ID_OK: - self.add_item(dlg.GetPath()) - dlg.Destroy() - e.Skip() - - def set_type_choices(self, path): - # Determine what type of input this is and present user with choices - # (this so far works for images ONLY) - type_choices = ['[ SELECT INPUT TYPE ]'] - preferred_selection = 0 - if os.path.isdir(path): - type_choices.extend(['processed pickle folder']) - dir_type = ginp.get_folder_type(path) - if dir_type in type_choices: - preferred_selection = type_choices.index(dir_type) - elif os.path.isfile(path): - file_type = ginp.get_file_type(path) - if file_type in ('processed pickle', 'reference MTZ', 'sequence'): - type_choices.extend(['processed pickle', - 'reference MTZ', - 'sequence']) - if file_type in type_choices: - preferred_selection = type_choices.index(file_type) - elif file_type == 'processed pickle list': - type_choices.extend(['processed pickle list']) - preferred_selection = type_choices.index('processed pickle list') - elif file_type in ('IOTA settings', - 'PRIME settings', - 'LABELIT target', - 'DIALS target'): - type_choices.extend(['IOTA settings', 'PRIME settings', - 'LABELIT target', 'DIALS target']) - preferred_selection = type_choices.index(file_type) - - return type_choices, preferred_selection - - def add_item(self, path): - # Generate item - inp_choices, inp_sel = self.set_type_choices(path) - type_choice = ct.DataTypeChoice(self.ctr, - choices=inp_choices) - item = ct.InputListItem(path=path, - type=type_choice, - buttons=ct.MiniButtonBoxInput(self.ctr)) - - self.Bind(wx.EVT_CHOICE, self.onTypeChoice, item.type.type) - item.buttons.btn_mag.Bind(wx.EVT_BUTTON, self.onMagButton) - item.buttons.btn_delete.Bind(wx.EVT_BUTTON, self.onDelButton) - item.buttons.btn_info.Bind(wx.EVT_BUTTON, self.onInfoButton) - - # Insert list item - idx = self.ctr.InsertStringItem(self.ctr.GetItemCount() + 1, item.path) - self.ctr.SetItemWindow(idx, 1, item.type, expand=True) - self.ctr.SetItemWindow(idx, 2, item.buttons, expand=True) - - # Set drop-down selection, check it for data and open other tabs - item.type.type.SetSelection(inp_sel) - if item.type.type.GetString(inp_sel) in ['processed pickle', - 'processed pickle list', - 'processed pickle folder']: - self.main_window.toolbar.EnableTool( - self.main_window.tb_btn_run.GetId(),True) - elif item.type.type.GetString(inp_sel) == 'reference MTZ': - self.input_window.opt_chk_useref.Enable() - elif item.type.type.GetString(inp_sel) == 'sequence': - pass - else: - warn_bmp = bitmaps.fetch_icon_bitmap('actions', 'status_unknown', - size=16) - item.buttons.btn_info.SetBitmapLabel(warn_bmp) - item.warning = True - - # Record index in all relevant places - item.id = idx - item.buttons.index = idx - item.type.index = idx - item.type_selection = inp_sel - - # Resize columns to fit content - col1_width = max(self.ctr.GetItemWindow(s, col=1).type.GetSize()[0] - for s in range(self.ctr.GetItemCount())) + 5 - col2_width = item.buttons.GetSize()[0] + 15 - col0_width = self.ctr.GetSize()[0] - col1_width - col2_width - self.ctr.SetColumnWidth(0, col0_width) - self.ctr.SetColumnWidth(1, col1_width) - self.ctr.SetColumnWidth(2, col2_width) - - # Make sure all the choice lists are the same size - if item.type.type.GetSize()[0] < col1_width - 5: - item.type.type.SetSize((col1_width - 5, -1)) - - # Attach data object to item - self.ctr.SetItemData(item.id, item) - - def onTypeChoice(self, e): - type = e.GetEventObject().GetParent() - item_data = self.ctr.GetItemData(type.index) - item_data.type.type.SetSelection(type.type.GetSelection()) - item_data.type_selection = type.type.GetSelection() - - # Evaluate whether data folders / files are present - data_items = 0 - for idx in range(self.ctr.GetItemCount()): - if self.ctr.GetItemData(idx).type_selection != 0: - data_items += 1 - if data_items > 0: - self.main_window.toolbar.EnableTool(self.main_window.tb_btn_run.GetId(), - True) - else: - self.main_window.toolbar.EnableTool(self.main_window.tb_btn_run.GetId(), - False) - e.Skip() - - def onMagButton(self, e): - idx = e.GetEventObject().GetParent().index - item_obj = self.ctr.GetItemData(idx) - path = item_obj.path - type = item_obj.type.type.GetString(item_obj.type_selection) - - if os.path.isfile(path): - if type in ('processed pickle list', 'sequence', 'text'): - with open(path, 'r') as f: - msg = f.read() - textview = dlg.TextFileView(self, title=path, contents=msg) - textview.ShowModal() - #TODO: when individual pickle, show pickle info; also include json files - # else: - # wx.MessageBox('Unknown file type', 'Warning', - # wx.OK | wx.ICON_EXCLAMATION) - elif os.path.isdir(path): - inputs, _ = ginp.get_input(path, filter_results=False) - file_list = '\n'.join(inputs) - filelistview = dlg.TextFileView(self, title=path, contents=file_list) - filelistview.ShowModal() - - def onDelButton(self, e): - item = e.GetEventObject().GetParent() - self.delete_button(item.index) - self.ctr.Refresh() - - def delete_all(self): - for idx in range(self.ctr.GetItemCount()): - self.delete_button(index=0) - - def delete_button(self, index): - item_data = self.ctr.GetItemData(index) - if item_data.type.type.GetString(item_data.type_selection) == 'reference MTZ': - self.input_window.opt_chk_useref.Disable() - self.input_window.opt_chk_useref.SetValue(False) - - self.ctr.DeleteItem(index) - - # Refresh widget and list item indices - if self.ctr.GetItemCount() != 0: - for i in range(self.ctr.GetItemCount()): - item_data = self.ctr.GetItemData(i) - item_data.id = i - item_data.buttons.index = i - item_data.type.index = i - type_choice = self.ctr.GetItemWindow(i, col=1) - type_selection = item_data.type.type.GetSelection() - type_choice.type.SetSelection(type_selection) - self.ctr.SetItemData(i, item_data) - - def onInfoButton(self, e): - ''' Info / alert / error button (will change depending on circumstance) ''' - idx = e.GetEventObject().GetParent().index - item_obj = self.ctr.GetItemData(idx) - item_type = item_obj.type.type.GetString(item_obj.type_selection) - - if item_obj.warning: - wx.MessageBox(item_obj.info['WARNING'], 'Warning', wx.OK | - wx.ICON_EXCLAMATION) - else: - wx.MessageBox(item_obj.info[item_type], 'Info', wx.OK | - wx.ICON_INFORMATION) diff --git a/prime/postrefine/mod_input.py b/prime/postrefine/mod_input.py deleted file mode 100644 index 51fac0c403b..00000000000 --- a/prime/postrefine/mod_input.py +++ /dev/null @@ -1,639 +0,0 @@ -from __future__ import division, print_function, absolute_import - -"""read PRIME input""" -#Define exceptions -class ReadInputError(Exception): pass -class InvalidData(ReadInputError): pass -class InvalidCrystalSystem(ReadInputError): pass -class InvalidPixelSize(ReadInputError): pass -class InvalidRunNo(ReadInputError): pass -class InvalidNumberOfResidues(ReadInputError): pass - -import iotbx.phil -from libtbx.utils import Usage, Sorry -import sys, os, shutil, glob, tarfile -from six.moves import cPickle as pickle - -master_phil = iotbx.phil.parse(""" -data = None - .type = path - .multiple = True - .help = Directory containing integrated data in pickle format. Repeat to \ - specify additional directories. - .style = input_list -run_no = None - .type = path - .help = Run no. is used as folder name that stores output files. - .optional = False - .alias = Output folder - .style = path:folder -title = None - .type = str - .help = Title of the run. - .multiple = False - .alias = Description -icering - .help = "Allowing exclusion of icering." - .alias = Exclude ice rings - .style = grid:auto has_scope_switch -{ - flag_on = False - .type = bool - .help = Turn this flag on to allow exclusion of icering. - .style = scope_switch - d_upper = 3.9 - .type = float - .help = Minimum resolution. - d_lower = 3.85 - .type = float - .help = Maximum resolution. -} -scale - .help = "Parameters used to generate mean-intensity scaled reference set." - .alias = Initial Scaling Parameters -{ - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI -} -postref - .help = Post-refinement parameters - .alias = Post-refinement Parameters -{ - residual_threshold = 5 - .type = float - .help = Percent increase in residual allowed during microcycle. - .alias = Residual threshold - residual_threshold_xy = 5 - .type = float - .help = Percent increase in residual (xy) allowed during microcycle. - .alias = Residual XY threshold - scale - .help = Scale factors - .style = grid:auto - .alias = Scale factors - { - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI - partiality_min = 0.1 - .type = float - .help = Minimum partiality cutoff. - .alias = Minimum partiality - } - crystal_orientation - .help = Crystal orientations - .alias = Crystal Orientation - .style = grid:auto has_scope_switch - .expert_level = 1 - { - flag_on = True - .type = bool - .help = Set to False to turn post-refinement in this section off. - .alias = Refine crystal orientation - .style = scope_switch - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI - partiality_min = 0.1 - .type = float - .help = Minimum partiality cutoff. - .alias = Minimum partiality - } - reflecting_range - .help = Reflecting range - .alias = Reflecting Range - .style = grid:auto has_scope_switch - .expert_level = 1 - { - flag_on = True - .type = bool - .help = Set to False to turn post-refinement in this section off. - .alias = Refine reflecting range - .style = scope_switch - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI - partiality_min = 0.1 - .type = float - .help = Minimum partiality cutoff. - .alias = Minimum partiality - } - unit_cell - .help = Unit-cell dimensions - .alias = Unit Cell Dimensions - .style = grid:auto has_scope_switch - .expert_level = 1 - { - flag_on = True - .type = bool - .help = Set to False to turn post-refinement in this section off. - .style = scope_switch - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI - partiality_min = 0.1 - .type = float - .help = Minimum partiality cutoff. - .alias = Minimum partiality - uc_tolerance = 5 - .type = float - .help = Unit-cell tolerance in percent. - .alias = Unit cell tolerance (%) - } - allparams - .help = All parameters - .alias = Refine All Parameters - .style = grid:auto has_scope_switch - .expert_level = 1 - { - flag_on = True - .type = bool - .help = Set to True to refine all parameters together. - .style = scope_switch - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI - partiality_min = 0.1 - .type = float - .help = Minimum partiality cutoff. - .alias = Minimum partiality - uc_tolerance = 5 - .type = float - .help = Unit-cell tolerance in percent. - .alias = Unit cell tolerance (%) - } -} -merge - .help = "Parameters used in merging" - .alias = Merging Parameters -{ - d_min = 0.1 - .type = float - .help = Minimum resolution. - .alias = High resolution limit - d_max = 99 - .type = float - .help = Maximum resolution. - .alias = Low resolution limit - sigma_min = -3.0 - .type = float - .help = Minimum I/sigI cutoff. - .help = Minimum I/sigI - partiality_min = 0.1 - .type = float - .help = Minimum partiality cutoff. - .alias = Minimum partiality - uc_tolerance = 5 - .type = float - .help = Unit-cell tolerance in percent. - .alias = Unit cell tolerance (%) - match_resolution = True - .type = bool - .help = GUI only: when True, merging resolution limits will be applied to \ - all post-refinement operations - .alias = Use general resolution limits -} -target_unit_cell = None - .type = unit_cell - .help = Target unit-cell parameters are used to discard outlier cells. - .optional = False - .alias = Target Unit Cell -flag_override_unit_cell = False - .type = bool - .help = Set to True to overide unit cell in observations with the target cell. -target_space_group = None - .type = space_group - .help = Target space group. - .optional = False - .alias = Target Space Group -target_anomalous_flag = False - .type = bool - .help = Target anomalous flag (False = Not anomalous data) - .optional = False - .alias = Anomalous -flag_weak_anomalous = False - .type = bool - .help = Set to True to indicate that you have weak anomalous signal. -target_crystal_system = None Triclinic Monoclinic Orthorhombic Tetragonal Trigonal Hexagonal Cubic - .type = choice - .help = Target crystal system - .optional = True - .alias = Crystal System -n_residues = None - .type = int - .help = No. of amino acid residues. - .alias = No. residues -indexing_ambiguity - .help = "Parameters used in resolving indexing ambiguity" -{ - mode = Auto - .type = str - .help = Set to Forced to solve pseudo-twinning. - index_basis_in = None - .type = path - .help = Pickle file with basis solution or an mtz file of an isomorphous structure. - assigned_basis = None - .multiple = True - .type = str - .help = Specify list of basis formats for pseudo-twinning. - d_min = 3.0 - .type = float - .help = In case index_basis_in given is an mtz file, you can pecify minimum resolution used to calculate correlation with the given mtz file. - d_max = 10.0 - .type = float - .help = In case index_basis_in given is an mtz file, you can pecify maximum resolution used to calculate correlation with the given mtz file. - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - n_sample_frames = 300 - .type = int - .help = No. of frames used in scoring r_matrix. Images (n_selected_frames) with the highest score will be used in the Brehm & Diederichs algorithm. - n_selected_frames = 100 - .type = int - .help = No. of frames used in Auto solution mode. The rest of the frame data will be determined against this merged dataset. -} -hklisoin = None - .type = path - .help = Mtz file for the calculation of CCiso -hklrefin = None - .type = path - .help = Mtz file used as a reference in post-refinement. - .expert_level = 1 -flag_plot = False - .type = bool - .help = Normal plots. - .expert_level = 1 -flag_plot_expert = False - .type = bool - .help = Expert plots. - .expert_level = 2 -n_postref_cycle = 3 - .type = int - .help = No. of cycles for post-refinement. -n_postref_sub_cycle = 1 - .type = int - .help = No. of cycles for the least-squares minimization in post-refinement. - .expert_level = 1 -n_rejection_cycle = 1 - .type = int - .help = No. of cycles for the outlier rejection. - .expert_level = 1 -sigma_rejection = 5 - .type = float - .help = Sigma level for outlier rejection. - .expert_level = 1 -n_bins = 20 - .type = int - .help = No. of bins used to report statistics. -pixel_size_mm = None - .type = float - .help = Pixel size in mm. (MAR = 0.079346) - .optional = False - .alias = Pixel Size -frame_accept_min_cc = 0.25 - .type = float - .help = CC cut-off for the rejection of frames before merging. -flag_apply_b_by_frame = True - .type = bool - .help = Set to False to dismiss B-factor checking. - .expert_level = 2 -flag_monte_carlo = False - .type = bool - .help = Set to True to turn on Monte-Carlo merge w/o partiality correction. Use n_postref_cycle=0 to output the merged mtz file without post-refinement and partiality correction. - .expert_level = 2 -b_refine_d_min = 99 - .type = float - .help = Minimum resolution. - .expert_level = 2 -partiality_model = Lorentzian - .type = str - .help = Your choice of partiality model: Lorentzian (default), Voigt (in beta test), Lognormal (in beta test). - .expert_level = 2 -flag_LP_correction = True - .type = bool - .help = Do polarization correction. - .expert_level = 2 -flag_volume_correction = True - .type = bool - .help = Do volume correction. - .expert_level = 2 -flag_beam_divergence = False - .type = bool - .help = Default is not to refine beam divergence. Set to True to allow gammaX and gammaY refinement. - .expert_level = 2 -n_processors = 32 - .type = int - .help = No. of processing units - .optional = True - .alias = No. processors -gamma_e = 0.003 - .type = float - .help = Initial spread of the energy spectrum (1/Angstrom). - .expert_level = 2 -voigt_nu = 0.5 - .type = float - .help = If select Voigt for partiality model, the voigt_nu parameter determines the Lorentzian and Gaussian component of the function (0.0 [Lorentzian]<= voigt_nu <= 1.0 [Gaussian]). The default value is 0.5 and will be refined in the run. - .expert_level = 2 -polarization_horizontal_fraction = 1.0 - .type = float - .help = Polarization fraction in horizontal direction. - .expert_level = 2 -flag_output_verbose = False - .type = bool - .help = Output full detail of the refinement results. - .expert_level = 1 -flag_replace_sigI = False - .type = bool - .help = Replace to experimental errors I with sqrt(I). - .expert_level = 2 -percent_cone_fraction = 5.0 - .type = float - .help = Perent used to select reflections inside a cone centering on each crystal axis. - .expert_level = 2 -isoform_name = None - .type = str - .help = Use this isoform. - .expert_level = 1 -flag_hush = False - .type = bool - .help = Set to true to hush all the disc and elaboarated stats. operations. - .expert_level = 1 -timeout_seconds = 300 - .type = int - .help = Time limits used when queing system is activated. - .expert_level = 1 - .alias = Queue timeout (sec) -queue - .help = "Parameters used for submitting jobs to queuing system." - .alias = Multiprocessing options -{ - mode = None - .type = str - .help = Queing system type. Only bsub is available now. - qname = psanaq - .type = str - .help = For system with queue name, specify your queue name here. For LCLS users, primary queue is the default value while high priority queue at NEH and FEH are psnehhiprioq and psfehhiprioq. - .alias = Queue - n_nodes = 12 - .type = int - .help = No. of nodes used. - .alias = No. of nodes -} -isoform_cluster - .help = "Parameters used in clustering isoform" - .alias = Isoform Clustering -{ - n_clusters = 2 - .type = int - .help = No. of expected isoforms. - .alias = No of isoforms - isorefin = None - .multiple = True - .type = path - .help = Specify list of mtz files for identifying isoform clusters. Note that n_clusters will be replaced with no. of items in this list. - .alias = Reference data (MTZ) - d_min = 3.0 - .type = float - .help = High resolution limit used in the calculation of r_metric. - .alias = High resolution limit - d_max = 10.0 - .type = float - .help = Low resolution limit used in the calculation of r_metric. - .alias = Low resolution limit - sigma_min = 1.5 - .type = float - .help = Minimum I/sigI cutoff. - .alias = Minimum I/sigI - n_sample_frames = 300 - .type = int - .help = No. of frames used in scoring r_matrix. Images (n_selected_frames) with the highest score will be used in the Brehm & Diederichs algorithm. - .alias = No. of sample frames - n_selected_frames = 100 - .type = int - .help = No. of frames used in Auto solution mode. The rest of the frame data will be determined against this merged dataset. - .alias = No. of selected frames -} -rejections = None - .type = str - .help = Dict of integration filenames and their rejected miller indices. -""") - -txt_help = """************************************************************************************************** - -Prime: post-refinement and merging. - -For more detail and citation, see Enabling X-ray free electron laser crystallography -for challenging biological systems from a limited number of crystals -"DOI: https://doi.org/10.7554/eLife.05421". - -Usage: prime.postrefine parameter.phil - -With this command, you can specify all parameters required by prime in your parameter.phil file. -To obtain the template of these parameters, you can perform a dry run (simply run prime.postrefine). -You can then change the values of the parameters. - -For feedback, please contact monarin@stanford.edu. - -************************************************************************************************** - -List of available parameters: -""" - -def process_input(argv=None, flag_mkdir=True): - user_phil = [] - if argv == None: - master_phil.show() - raise Usage("Use the above list of parameters to generate your input file (.phil). For more information, run prime.postrefine -h.") - else: - for arg in argv: - if os.path.isfile(arg): - user_phil.append(iotbx.phil.parse(open(arg).read())) - elif (os.path.isdir(arg)) : - user_phil.append(iotbx.phil.parse("""data=\"%s\" """ % arg)) - else : - if arg == '--help' or arg == '-h': - print (txt_help) - master_phil.show(attributes_level=1) - raise Usage("Run prime.run to generate a list of initial parameters.") - else: - try: - user_phil.append(iotbx.phil.parse(arg)) - except RuntimeError as e : - raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e))) - #setup phil parameters - working_phil = master_phil.fetch(sources=user_phil) - params = working_phil.extract() - if not params.data: - raise InvalidData("Error: Data is required. Please specify path to your data folder (data=/path/to/integration/results).") - - #check target_crystal_system - crystal_system_dict = {'Triclinic': 0, 'Monoclinic': 0, 'Orthorhombic': 0, 'Tetragonal': 0, 'Trigonal': 0, 'Hexagonal': 0, 'Cubic':0} - if params.target_crystal_system is not None: - if params.target_crystal_system not in crystal_system_dict: - raise InvalidCrystalSystem("Error: Invalid input target_crystal_system. Please choose following options: Triclinic, Monoclinic, Orthorhombic, Tetragonal, Trigonal, Hexagonal, or Cubic.") - - #check n_residues - if not params.n_residues: - raise InvalidNumberOfResidues("Error: Number of residues is required. Please specify number of residues of your structure in asymmetric unit (n_residues = xxx).") - - #check pixel_size - if not params.pixel_size_mm: - #look in the new integration pickle format (2016-08-05) - try: - frame_files = read_pickles(params.data) - frame_0 = frame_files[0] - int_pickle = read_frame(frame_0) - params.pixel_size_mm = int_pickle['pixel_size'] - print ('Info: Found pixel size in the integration pickles (override pixel_size_mm=%10.8f)'%(params.pixel_size_mm)) - except Exception: - raise InvalidPixelSize("Error: Pixel size in millimeter is required. Use cctbx.image_viewer to view one of your images and note down the value (e.g. for marccd, set pixel_size_mm=0.079346).") - - #check sigma rejection - if params.sigma_rejection < 5: - print("Warning: sigma below 5 will result in discarding too many reflections") - - #generate run_no folder - if not params.run_no: - #use default name - default_run = 'Prime_Run_' - all_runs = glob.glob(default_run+'*') - new_run_no = 1 - if all_runs: new_run_no = max([int(run_no.split('_')[-1]) for run_no in all_runs])+1 - params.run_no = default_run+str(new_run_no) - elif os.path.exists(params.run_no): - print ("Warning: run number %s already exists."%(params.run_no)) - run_overwrite = raw_input('Overwrite?: N/Y (Enter for default)') - if run_overwrite == 'Y': - shutil.rmtree(params.run_no) - else: - raise InvalidRunNo("Error: Run number exists. Please specifiy different run no.") - - #make result folders - if flag_mkdir: - os.makedirs(params.run_no) - os.makedirs(params.run_no+'/index_ambiguity') - os.makedirs(params.run_no+'/isoform_cluster') - os.makedirs(params.run_no+'/stats') - - #capture input read out by phil - from six.moves import cStringIO as StringIO - class Capturing(list): - def __enter__(self): - self._stdout = sys.stdout - sys.stdout = self._stringio = StringIO() - return self - def __exit__(self, *args): - self.extend(self._stringio.getvalue().splitlines()) - sys.stdout = self._stdout - - with Capturing() as output: - working_phil.show() - - txt_out = 'prime.postrefine input:\n' - for one_output in output: - txt_out += one_output + '\n' - - return params, txt_out - -def read_pickles(data): - frame_files = [] - tar_files = [] - for p in data: - is_tar = False - if p.find('tar') >= 0: is_tar = True - if os.path.isdir(p) == False: - if os.path.isfile(p): - #read all file paths in the given input file - with open(p,'r') as f: file_list = f.read().splitlines() - else: - # p is a glob - file_list = glob.glob(p) - else: - file_list = glob.glob(os.path.join(p,'*')) - frame_files.extend(file_list) - if len(frame_files) == 0: - raise InvalidData("Error: no integration results found in the specified data parameter.") - if not is_tar: return frame_files - #take care of tar files - for tar_filename in frame_files: - tarf = tarfile.open(name=tar_filename, mode='r') - for myindex in range(len(tarf.getmembers())): - tar_files.append(tar_filename+':ind'+str(myindex)) - return tar_files - -def read_frame(frame_file): - ''' - A frame_file can be .pickle or .tar:ind#no. - Read accordingly and return integration pickle - ''' - observations_pickle = None - try: - if frame_file.endswith('.pickle'): - observations_pickle = pickle.load(open(frame_file,"rb")) - else: - tar_filename, tar_index = frame_file.split(':ind') - tarf = tarfile.open(name=tar_filename, mode='r') - tar_member = tarf.extractfile(member=tarf.getmembers()[int(tar_index)]) - observations_pickle = pickle.load(tar_member) - except Exception: - print ("Warning: unable to read %s"%(frame_file)) - pass - if any([len(obs.data()) == 0 for obs in observations_pickle['observations']]): - print ("Empty file %s"%(frame_file)) - return - return observations_pickle diff --git a/prime/postrefine/mod_lbfgs.py b/prime/postrefine/mod_lbfgs.py deleted file mode 100644 index 50aefb92ab1..00000000000 --- a/prime/postrefine/mod_lbfgs.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import absolute_import, division, print_function -from scitbx import lbfgs -from cctbx.array_family import flex -from .mod_lbfgs_partiality import lbfgs_partiality_handler -from six.moves import range - -class lbfgs_handler(object): - """ - lbfgs_handler optimize set of parameters (params) by fitting - data[0] to data [1] using given function (func). - """ - def __init__(self, current_x=None, args=None, - min_iterations=0, max_iterations=None, max_calls=1000, max_drop_eps=1.e-5): - self.n = current_x.size() - self.x = current_x - self.args = args - self.minimizer = lbfgs.run( - target_evaluator=self, - termination_params=lbfgs.termination_parameters( - traditional_convergence_test=False, - drop_convergence_test_max_drop_eps=max_drop_eps, - min_iterations=min_iterations, - max_iterations=max_iterations, - max_calls=max_calls), - exception_handling_params=lbfgs.exception_handling_parameters( - ignore_line_search_failed_rounding_errors=True, - ignore_line_search_failed_step_at_lower_bound=True, - ignore_line_search_failed_step_at_upper_bound=False, - ignore_line_search_failed_maxfev=False, - ignore_line_search_failed_xtol=False, - ignore_search_direction_not_descent=False) - ) - - def compute_functional_and_gradients(self): - lp_h = lbfgs_partiality_handler() - #calculate sum_sqr of the function - fvec = lp_h.func(self.x, self.args) - self.f = flex.sum(fvec*fvec) - #calculate gradient for each parameter - DELTA = 1.E-7 - self.g = flex.double() - for x in range(self.n): - templist = list(self.x) - templist[x]+=DELTA - dvalues = flex.double(templist) - dfvec = lp_h.func(dvalues, self.args) - df = flex.sum(dfvec*dfvec) - #calculate by finite_difference - self.g.append( ( df-self.f )/DELTA ) - return self.f, self.g diff --git a/prime/postrefine/mod_lbfgs_partiality.py b/prime/postrefine/mod_lbfgs_partiality.py deleted file mode 100644 index 88210b90df0..00000000000 --- a/prime/postrefine/mod_lbfgs_partiality.py +++ /dev/null @@ -1,113 +0,0 @@ -from __future__ import absolute_import, division, print_function -from cctbx.array_family import flex -from cctbx.uctbx import unit_cell -from .mod_partiality import partiality_handler -""" -lbfgs_partiality_handler -calculate the minimization function according to given arguments and return sum(error^2) -""" -class lbfgs_partiality_handler(object): - """ - lbfgs_handler optimizes set of parameters (params) by fitting - data[0] to data [1] using given function (func). - """ - def __init__(self): - """ - Intialitze parameters - """ - - def prep_input(self, params, cs): - #From crystal system cs, determine refined parameters - a, b, c, alpha, beta, gamma = params - if cs == 'Triclinic': - x0 = params - elif cs == 'Monoclinic': - x0 = flex.double([a,b,c,beta]) - elif cs == 'Orthorhombic': - x0 = flex.double([a,b,c]) - elif cs == 'Tetragonal': - x0 = flex.double([a,c]) - elif cs == 'Trigonal' or cs == 'Hexagonal': - x0 = flex.double([a,c]) - elif cs == 'Cubic': - x0 = flex.double([a]) - return x0 - - def prep_output(self, params, cs): - if cs == 'Triclinic': - xopt = params - elif cs == 'Monoclinic': - xopt = flex.double([params[0],params[1],params[2],90,params[3],90]) - elif cs == 'Orthorhombic': - xopt = flex.double([params[0],params[1],params[2],90,90,90]) - elif cs == 'Tetragonal': - xopt = flex.double([params[0],params[0],params[1],90,90,90]) - elif cs == 'Trigonal' or cs == 'Hexagonal': - xopt = flex.double([params[0],params[0],params[1],90,90,120]) - elif cs == 'Cubic': - xopt = flex.double([params[0],params[0],params[0],90,90,90]) - return xopt - - def func(self, params, args): - I_r = args[0] - miller_array_o = args[1] - wavelength = args[2] - alpha_angle_set = args[3] - crystal_init_orientation = args[4] - spot_pred_x_mm_set = args[5] - spot_pred_y_mm_set = args[6] - detector_distance_mm = args[7] - refine_mode = args[8] - const_params = args[9] - b0 = args[10] - miller_array_iso = args[11] - iparams = args[12] - partiality_model = iparams.partiality_model - flag_volume_correction = iparams.flag_volume_correction - flag_beam_divergence = iparams.flag_beam_divergence - b_refine_d_min = iparams.b_refine_d_min - I_o = miller_array_o.data() - sigI_o = miller_array_o.sigmas() - miller_indices_original = miller_array_o.indices() - sin_theta_over_lambda_sq = miller_array_o.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() - two_theta_flex = miller_array_o.two_theta(wavelength=wavelength).data() - cs = miller_array_o.crystal_symmetry().space_group().crystal_system() - if refine_mode == 'scale_factor': - G, B = params - rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params - elif refine_mode == 'crystal_orientation': - rotx, roty = params - G, B, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params - elif refine_mode == 'reflecting_range': - ry, rz, r0, re, nu = params - G, B, rotx, roty, a, b, c, alpha, beta, gamma = const_params - elif refine_mode == 'unit_cell': - a, b, c, alpha, beta, gamma = self.prep_output(params, cs) - G, B, rotx, roty, ry, rz, r0, re, nu = const_params - elif refine_mode == 'allparams': - a, b, c, alpha, beta, gamma = self.prep_output(params[7:], cs) - rotx, roty, ry, rz, r0, re, nu = params[0:7] - G,B = const_params - try: - uc = unit_cell((a,b,c,alpha,beta,gamma)) - except Exception: - return None - G,B,rotx,roty,ry,rz,r0,re,nu,a,b,c,alpha,beta,gamma = flex.double([G,B,rotx,roty,ry,rz,r0,re,nu,a,b,c,alpha,beta,gamma]) - ph = partiality_handler() - p_calc_flex, delta_xy_flex, rs_set, dummy = ph.calc_partiality_anisotropy_set(\ - uc, rotx, roty, miller_indices_original, ry, rz, r0, re, nu, two_theta_flex, - alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set, - spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence) - if miller_array_o.d_min() < b_refine_d_min: - I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, B, p_calc_flex, rs_set, flag_volume_correction) - else: - I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, b0, p_calc_flex, rs_set, flag_volume_correction) - if refine_mode == 'unit_cell': - error = delta_xy_flex - else: - error = ((I_r - I_o_full)/sigI_o) - """ - import math - print refine_mode, 'G:%.4g B:%.4g rotx:%.4g roty:%.4g r0:%.4g re:%.4g nu:%6.4f a:%.4g b:%.4g c:%.4g fpr:%.4g fxy:%.4g n_refl=%5.0f'%(G, B, rotx*180/math.pi, roty*180/math.pi, r0, re, nu, a, b, c, flex.sum(((I_r - I_o_full)/sigI_o)**2), flex.sum(delta_xy_flex**2), len(I_o_full)) - """ - return error diff --git a/prime/postrefine/mod_leastsqr.py b/prime/postrefine/mod_leastsqr.py deleted file mode 100644 index ef61cb33cec..00000000000 --- a/prime/postrefine/mod_leastsqr.py +++ /dev/null @@ -1,470 +0,0 @@ -''' -Author : Uervirojnangkoorn, M. -Created : 7/13/2014 -Description : - -The leastsqr_handler class refines post-refinement parameters for each frame m in: - -J = sum(I_hi - (G0 * exp(-2B*sin_theta_over_lambda_sq) * p * I_h))^2 - -where I_hi is observed intensitiy, G0 and B are scale factors, and p -is the partiality function of these parameters: -rot_x, rot_y: crystal orientation angles around x- and y- axes -gamma_y, gamma_z: parameters to model anisotropy of crystal mosaicity -gamma_e: spectral dispersion -unit-cell parameters: a,b,c,alpha,beta,gamma. - -The class implements Leveberg-Marquardt algorithms to scale the refined parameters -using the lamda updates. The unit-cell parameters are refined with restraints based -on the 7 crystal systems (6 conditions). -''' -from __future__ import absolute_import, division, print_function -import math -from cctbx.array_family import flex -from scitbx.matrix import sqr -from cctbx.uctbx import unit_cell -from cctbx.crystal_orientation import crystal_orientation -from .mod_lbfgs import lbfgs_handler -from .mod_lbfgs_partiality import lbfgs_partiality_handler -from .mod_partiality import partiality_handler -from six.moves import range - -def coefficient_of_determination(y, y_model): - mean_y = flex.mean(y) - r_sqr = flex.sum((y_model - mean_y)**2)/flex.sum((y - mean_y)**2) - return r_sqr - -def standard_error_of_the_estimate(y, y_model, n_params): - s = math.sqrt(flex.sum((y - y_model)**2)/(len(y) - n_params)) - return s - -def good_unit_cell(uc_params, iparams, uc_tol, target_unit_cell=None): - if iparams is None: - expected_unit_cell = target_unit_cell - else: - expected_unit_cell = iparams.target_unit_cell - flag_good_uc = False - if (abs(uc_params[0]-expected_unit_cell.parameters()[0]) \ - <= (uc_tol*expected_unit_cell.parameters()[0]/100) \ - and abs(uc_params[1]-expected_unit_cell.parameters()[1]) \ - <= (uc_tol*expected_unit_cell.parameters()[1]/100) \ - and abs(uc_params[2]-expected_unit_cell.parameters()[2]) \ - <= (uc_tol*expected_unit_cell.parameters()[2]/100) \ - and abs(uc_params[3]-expected_unit_cell.parameters()[3]) \ - <= (uc_tol*expected_unit_cell.parameters()[3]/100) \ - and abs(uc_params[4]-expected_unit_cell.parameters()[4]) \ - <= (uc_tol*expected_unit_cell.parameters()[4]/100) \ - and abs(uc_params[5]-expected_unit_cell.parameters()[5]) \ - <= (uc_tol*expected_unit_cell.parameters()[5]/100)): - flag_good_uc = True - return flag_good_uc - -class leastsqr_handler(object): - """ - A wrapper class for least-squares refinement - """ - def __init__(self): - """ - Intialitze parameters - """ - - def get_filtered_data(self, filter_mode, filter_params, - observations_in, alpha_angle_in, - spot_pred_x_mm_in, spot_pred_y_mm_in, - I_ref_in, partiality_in=False): - if filter_mode == 'resolution': - i_sel = observations_in.resolution_filter_selection(d_min=filter_params[0],\ - d_max=filter_params[1]) - elif filter_mode == 'sigma': - i_sel = (observations_in.data()/observations_in.sigmas()) > filter_params[0] - elif filter_mode == 'partiality': - i_sel = partiality_in > filter_params[0] - observations_out = observations_in.select(i_sel) - alpha_angle_out = alpha_angle_in.select(i_sel) - spot_pred_x_mm_out = spot_pred_x_mm_in.select(i_sel) - spot_pred_y_mm_out = spot_pred_y_mm_in.select(i_sel) - I_ref_out = I_ref_in.select(i_sel) - return observations_out, alpha_angle_out, spot_pred_x_mm_out, spot_pred_y_mm_out, I_ref_out - - def optimize_scalefactors(self, I_r_flex, observations_original, - wavelength, crystal_init_orientation, - alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, - pres_in, observations_non_polar, detector_distance_mm, const_params): - ph = partiality_handler() - pr_d_min = iparams.postref.scale.d_min - pr_d_max = iparams.postref.scale.d_max - pr_sigma_min = iparams.postref.scale.sigma_min - pr_partiality_min = iparams.postref.scale.partiality_min - #filter by resolution - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\ - spot_pred_x_mm, spot_pred_y_mm, I_r_flex) - #filter by sigma - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\ - spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) - I_r_true = I_ref_sel[:] - I_o_true = observations_original_sel.data()[:] - if pres_in is not None: - G, B, b0 = pres_in.G, pres_in.B, pres_in.B - else: - G,B,b0 = (1,0,0) - refine_mode = 'scale_factor' - xinp = flex.double([G,B]) - args = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, - crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, - detector_distance_mm, refine_mode, const_params, b0, None, iparams) - lh = lbfgs_handler(current_x=xinp, args=args) - G_fin, B_fin = (lh.x[0], lh.x[1]) - rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params - two_theta = observations_original.two_theta(wavelength=wavelength) - sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data() - uc = unit_cell((a,b,c,alpha,beta,gamma)) - ph = partiality_handler() - partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty, - observations_original.indices(), - ry, rz, r0, re, voigt_nu, - two_theta.data(), - alpha_angle, - wavelength, - crystal_init_orientation, - spot_pred_x_mm, - spot_pred_y_mm, - detector_distance_mm, - iparams.partiality_model, - iparams.flag_beam_divergence) - I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, - G, B, partiality_init, rs_init) - I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, - G_fin, B_fin, partiality_init, rs_init) - SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex, I_o_fin, 2) - R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100 - CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient() - CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient() - err_init = (I_r_flex - I_o_init)/observations_original.sigmas() - R_init = math.sqrt(flex.sum(err_init**2)) - err_final = (I_r_flex - I_o_fin)/observations_original.sigmas() - R_final = math.sqrt(flex.sum(err_final**2)) - R_xy_init = 0 - R_xy_final = 0 - CC_iso_init = 0 - CC_iso_final = 0 - return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final) - - def prepare_data_microcycle(self, refine_mode, iparams, - observations_original, alpha_angle, - spot_pred_x_mm, spot_pred_y_mm, - I_r_flex, init_params, crystal_init_orientation, - wavelength, detector_distance_mm): - #prepare data - if refine_mode == 'crystal_orientation': - pr_d_min = iparams.postref.crystal_orientation.d_min - pr_d_max = iparams.postref.crystal_orientation.d_max - pr_sigma_min = iparams.postref.crystal_orientation.sigma_min - pr_partiality_min = iparams.postref.crystal_orientation.partiality_min - pr_uc_tol = iparams.postref.unit_cell.uc_tolerance - elif refine_mode == 'reflecting_range': - pr_d_min = iparams.postref.reflecting_range.d_min - pr_d_max = iparams.postref.reflecting_range.d_max - pr_sigma_min = iparams.postref.reflecting_range.sigma_min - pr_partiality_min = iparams.postref.reflecting_range.partiality_min - pr_uc_tol = iparams.postref.unit_cell.uc_tolerance - elif refine_mode == 'unit_cell': - pr_d_min = iparams.postref.unit_cell.d_min - pr_d_max = iparams.postref.unit_cell.d_max - pr_sigma_min = iparams.postref.unit_cell.sigma_min - pr_partiality_min = iparams.postref.unit_cell.partiality_min - pr_uc_tol = iparams.postref.unit_cell.uc_tolerance - elif refine_mode == 'allparams': - pr_d_min = iparams.postref.allparams.d_min - pr_d_max = iparams.postref.allparams.d_max - pr_sigma_min = iparams.postref.allparams.sigma_min - pr_partiality_min = iparams.postref.allparams.partiality_min - pr_uc_tol = iparams.postref.unit_cell.uc_tolerance - #filter by resolution - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle, - spot_pred_x_mm, spot_pred_y_mm, I_r_flex) - #filter by sigma - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel, - spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) - #extract refined parameters - G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = init_params - #filter by partiality - two_theta = observations_original_sel.two_theta(wavelength=wavelength).data() - uc = unit_cell((a,b,c,alpha,beta,gamma)) - ph = partiality_handler() - partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty, - observations_original_sel.indices(), - ry, rz, r0, re, voigt_nu, two_theta, - alpha_angle_sel, wavelength, - crystal_init_orientation, - spot_pred_x_mm_sel, - spot_pred_y_mm_sel, - detector_distance_mm, - iparams.partiality_model, - iparams.flag_beam_divergence) - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'partiality', [pr_partiality_min], observations_original_sel, - alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel, - partiality_in=partiality_init) - return observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel - - def optimize(self, I_r_flex, observations_original, - wavelength, crystal_init_orientation, - alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, - pres_in, observations_non_polar, detector_distance_mm): - ph = partiality_handler() - lph = lbfgs_partiality_handler() - if iparams.postref.allparams.flag_on: - refine_steps = ['allparams'] - else: - refine_steps = ['crystal_orientation'] - if iparams.postref.reflecting_range.flag_on: - refine_steps.append('reflecting_range') - if iparams.postref.unit_cell.flag_on: - refine_steps.append('unit_cell') - #get miller array iso, if given. - miller_array_iso = None - #prepare data - pr_d_min = iparams.postref.allparams.d_min - pr_d_max = iparams.postref.allparams.d_max - pr_sigma_min = iparams.postref.allparams.sigma_min - pr_partiality_min = iparams.postref.allparams.partiality_min - pr_uc_tol = iparams.postref.allparams.uc_tolerance - cs = observations_original.crystal_symmetry().space_group().crystal_system() - #filter by resolution - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle, - spot_pred_x_mm, spot_pred_y_mm, I_r_flex) - #filter by sigma - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel, - spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) - #initialize values only in the first sub cycle and the first refine step. - spot_radius = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), - observations_original_sel.indices(), wavelength) - if pres_in is None: - ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0 - #apply constrain on the unit cell using crystal system - uc_scale_inp = lph.prep_input(observations_original.unit_cell().parameters(), cs) - uc_scale_constrained = lph.prep_output(uc_scale_inp, cs) - a,b,c,alpha,beta,gamma = uc_scale_constrained - const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) - xopt_scalefactors, stats = self.optimize_scalefactors(I_r_flex, - observations_original, - wavelength, crystal_init_orientation, - alpha_angle, - spot_pred_x_mm, - spot_pred_y_mm, - iparams, - pres_in, - observations_non_polar, - detector_distance_mm, - const_params_scale) - G, B = xopt_scalefactors - else: - G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0 , 0.0 - a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters() - crystal_init_orientation = pres_in.crystal_orientation - #filter by partiality - two_theta = observations_original_sel.two_theta(wavelength=wavelength).data() - uc = unit_cell((a,b,c,alpha,beta,gamma)) - partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty, - observations_original_sel.indices(), - ry, rz, r0, re, voigt_nu, two_theta, - alpha_angle_sel, wavelength, - crystal_init_orientation, - spot_pred_x_mm_sel, - spot_pred_y_mm_sel, - detector_distance_mm, - iparams.partiality_model, - iparams.flag_beam_divergence) - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ - 'partiality', [pr_partiality_min], observations_original_sel, - alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel, - partiality_in=partiality_init) - I_r_true = I_ref_sel[:] - I_o_true = observations_original_sel.data()[:] - #calculate initial residual_xy error - const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) - xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs) - args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, - crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, - detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams) - uc_params_err = lph.func(xinp_uc, args_uc) - init_residual_xy_err = flex.sum(uc_params_err**2) - #calculate initial residual_pr error - const_params_all= (G,B) - xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) - xinp_all.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs)) - args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, - crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, - detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams) - all_params_err = lph.func(xinp_all, args_all) - init_residual_err = flex.sum(all_params_err**2) - #keep in list - t_pr_list = [init_residual_err] - t_xy_list = [init_residual_xy_err] - refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)] - txt_out = '' - for i_sub_cycle in range(iparams.n_postref_sub_cycle): - for j_refine_step in range(len(refine_steps)): - refine_mode = refine_steps[j_refine_step] - #prepare data - init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) - observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ - spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams, - observations_original, alpha_angle, - spot_pred_x_mm, spot_pred_y_mm, - I_r_flex, init_params, crystal_init_orientation, - wavelength, detector_distance_mm) - I_r_true = I_ref_sel[:] - I_o_true = observations_original_sel.data() - if refine_mode == 'crystal_orientation': - xinp = flex.double([rotx, roty]) - const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) - elif refine_mode == 'reflecting_range': - xinp = flex.double([ry, rz, r0, re, voigt_nu]) - const_params = (G, B, rotx, roty, a, b, c, alpha, beta, gamma) - elif refine_mode == 'unit_cell': - xinp = lph.prep_input((a,b,c,alpha,beta,gamma), cs) - const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) - elif refine_mode == 'allparams': - xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) - xinp.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs)) - const_params = (G,B) - args=(I_r_true, observations_original_sel, wavelength, alpha_angle_sel, - crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, - detector_distance_mm, refine_mode, const_params, B, miller_array_iso, iparams) - lh = lbfgs_handler(current_x=xinp, args=args) - xopt = flex.double(list(lh.x)) - if refine_mode == 'crystal_orientation' or \ - refine_mode == 'reflecting_range' or refine_mode == 'allparams': - current_residual_err = lh.f - #calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params) - xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs) - if refine_mode == 'crystal_orientation': - rotx, roty = xopt - elif refine_mode == 'reflecting_range': - ry, rz, r0, re, voigt_nu = xopt - elif refine_mode == 'allparams': - rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7] - xinp_uc = xopt[7:] - a, b, c, alpha, beta, gamma = lph.prep_output(xinp_uc, cs) - const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) - xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs) - args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, - crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, - detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams) - uc_params_err = lph.func(xinp_uc, args_uc) - current_residual_xy_err = flex.sum(uc_params_err**2) - elif refine_mode == 'unit_cell': - current_residual_xy_err = lh.f - xopt_uc = lph.prep_output(xopt, cs) - a, b, c, alpha, beta, gamma = xopt_uc - #check the unit-cell with the reference intensity - xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) - xinp.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs)) - const_params_all = (G,B) - args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, - crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, - detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams) - all_params_err = lph.func(xinp_all, args_all) - current_residual_err = flex.sum(all_params_err**2) - flag_success = False - if refine_mode == 'allparams': - #if allparams refinement, only check the post-refine target function - if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \ - (t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)): - t_pr_list.append(current_residual_err) - t_xy_list.append(current_residual_xy_err) - refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)) - flag_success = True - else: - if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \ - (t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)): - if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \ - (t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)): - t_pr_list.append(current_residual_err) - t_xy_list.append(current_residual_xy_err) - refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)) - flag_success = True - if flag_success is False: - G,B,rotx,roty,ry,rz,r0,re,voigt_nu,a,b,c,alpha,beta,gamma = refined_params_hist[len(refined_params_hist)-1] - tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n'%(i_sub_cycle,G,B,rotx*180/math.pi,roty*180/math.pi,ry,rz,r0,re,voigt_nu,a,c,t_pr_list[len(t_pr_list)-1],t_xy_list[len(t_pr_list)-1]) - txt_out += tmp_txt_out - #apply the refined parameters on the full (original) reflection set - two_theta = observations_original.two_theta(wavelength=wavelength).data() - sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() - if pres_in is None: - partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ - observations_original.unit_cell(),0.0, 0.0,observations_original.indices(), - 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, - two_theta, alpha_angle, wavelength, - crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, - iparams.partiality_model,iparams.flag_beam_divergence) - I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, - 1, 0, partiality_init, rs_init) - else: - partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ - pres_in.unit_cell,0.0, 0.0,observations_original.indices(), - pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu, - two_theta, alpha_angle, wavelength, - crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, - iparams.partiality_model,iparams.flag_beam_divergence) - I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, - pres_in.G, pres_in.B, partiality_init, rs_init) - partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\ - unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(), - ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation, - spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, - iparams.partiality_model,iparams.flag_beam_divergence) - I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, - G, B, partiality_fin, rs_fin) - SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex,I_o_fin, 13) - R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100 - CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient() - CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient() - err_init = (I_r_flex - I_o_init)/observations_original.sigmas() - R_init = math.sqrt(flex.sum(err_init**2)) - err_final = (I_r_flex - I_o_fin)/observations_original.sigmas() - R_final = math.sqrt(flex.sum(err_final**2)) - R_xy_init = math.sqrt(flex.sum(delta_xy_init**2)) - R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2)) - if R_init < R_final or re > (iparams.gamma_e * 3): - CC_final = CC_init - R_final = R_init - R_xy_final = R_xy_init - if pres_in is None: - G,B,r0,ry,rz,re,rotx,roty = (1.0,0.0,spot_radius,0.0,0.0,iparams.gamma_e,0.0,0.0) - a,b,c,alpha,beta,gamma = observations_original.unit_cell().parameters() - else: - G,B,r0,ry,rz,re,rotx,roty = (pres_in.G,pres_in.B,pres_in.r0,pres_in.ry,pres_in.rz,pres_in.re,0.0,0.0) - a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters() - crystal_init_orientation = pres_in.crystal_orientation - #calculate CCiso if hklisoin is given - CC_iso_init,CC_iso_final = (0,0) - if iparams.hklisoin is not None: - if miller_array_iso is not None: - from cctbx import miller - matches = miller.match_multi_indices( - miller_indices_unique=miller_array_iso.indices(), - miller_indices=observations_non_polar.indices()) - I_iso_match = flex.double([miller_array_iso.data()[pair[0]] for pair in matches.pairs()]) - I_o_init_match = flex.double([I_o_init[pair[1]] for pair in matches.pairs()]) - I_o_fin_match = flex.double([I_o_fin[pair[1]] for pair in matches.pairs()]) - CC_iso_init = flex.linear_correlation(I_iso_match, I_o_init_match).coefficient() - CC_iso_final = flex.linear_correlation(I_iso_match, I_o_fin_match).coefficient() - xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,b,c,alpha,beta,gamma) - return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final), len(I_ref_sel) diff --git a/prime/postrefine/mod_merge_data.py b/prime/postrefine/mod_merge_data.py deleted file mode 100644 index 7bcbd2ed101..00000000000 --- a/prime/postrefine/mod_merge_data.py +++ /dev/null @@ -1,165 +0,0 @@ -from __future__ import absolute_import, division, print_function -from cctbx.array_family import flex -from cctbx import miller -import math -from six.moves import range - -class merge_data_handler(object): - """keep arrays of unmerged data""" - def __init__(self): - #extract data - self.miller_indices_merge = flex.miller_index() - self.I_merge = flex.double() - self.sigI_merge = flex.double() - self.r_meas_div = flex.double() - self.r_meas_divisor = flex.double() - self.multiplicities = flex.int() - self.I_even = flex.double() - self.I_odd = flex.double() - self.I_even_h = flex.double() - self.I_odd_h = flex.double() - self.I_even_k = flex.double() - self.I_odd_k = flex.double() - self.I_even_l = flex.double() - self.I_odd_l = flex.double() - self.miller_array_merge = None - - def extend_data(self, miller_indices_merge, I_merge, sigI_merge, stat_all, - I_two_halves_tuple, uc_mean, wavelength_mean): - self.miller_indices_merge.extend(miller_indices_merge) - self.I_merge.extend(I_merge) - self.sigI_merge.extend(sigI_merge) - r_meas_div, r_meas_divisor, multiplicities = stat_all - self.r_meas_div.extend(r_meas_div) - self.r_meas_divisor.extend(r_meas_divisor) - self.multiplicities.extend(multiplicities) - I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple - self.I_even.extend(I_even) - self.I_odd.extend(I_odd) - self.I_even_h.extend(I_even_h) - self.I_odd_h.extend(I_odd_h) - self.I_even_k.extend(I_even_k) - self.I_odd_k.extend(I_odd_k) - self.I_even_l.extend(I_even_l) - self.I_odd_l.extend(I_odd_l) - self.uc_mean = uc_mean - self.wavelength_mean = wavelength_mean - - def extend(self, mdh): - self.miller_indices_merge.extend(mdh.miller_indices_merge) - self.I_merge.extend(mdh.I_merge) - self.sigI_merge.extend(mdh.sigI_merge) - self.r_meas_div.extend(mdh.r_meas_div) - self.r_meas_divisor.extend(mdh.r_meas_divisor) - self.multiplicities.extend(mdh.multiplicities) - self.I_even.extend(mdh.I_even) - self.I_odd.extend(mdh.I_odd) - self.I_even_h.extend(mdh.I_even_h) - self.I_odd_h.extend(mdh.I_odd_h) - self.I_even_k.extend(mdh.I_even_k) - self.I_odd_k.extend(mdh.I_odd_k) - self.I_even_l.extend(mdh.I_even_l) - self.I_odd_l.extend(mdh.I_odd_l) - self.uc_mean = mdh.uc_mean - self.wavelength_mean = mdh.wavelength_mean - - def generate_miller_array_from_miller_set(self, miller_set, target_anomalous_flag): - self.miller_array_merge = miller_set.array() \ - .customized_copy(indices=self.miller_indices_merge, \ - data=self.I_merge, sigmas=self.sigI_merge, anomalous_flag=target_anomalous_flag) \ - .map_to_asu() \ - .set_observation_type_xray_intensity() - return self.miller_array_merge - - def get_size(self): - return len(self.I_merge) - - def get_cc_anom(self): - cc_anom_acentric, nrefl_anom_acentric = (0,0) - if self.miller_array_merge.anomalous_flag(): - ma_anom_dif_even = self.miller_array_merge.customized_copy(data = self.I_even).anomalous_differences() - ma_anom_dif_odd = self.miller_array_merge.customized_copy(data = self.I_odd).anomalous_differences() - i_acentric = (ma_anom_dif_even.centric_flags().data() == False) - cc_anom_acentric = flex.linear_correlation(ma_anom_dif_even.data().select(i_acentric), ma_anom_dif_odd.data().select(i_acentric)).coefficient() - nrefl_anom_acentric = len(ma_anom_dif_even.data().select(i_acentric)) - return cc_anom_acentric, nrefl_anom_acentric - - def get_multiplicity(self): - return flex.sum(self.multiplicities)/self.get_size() if self.get_size() else 0 - - def get_r_meas(self): - return flex.sum(self.r_meas_div)/ flex.sum(self.r_meas_divisor) if flex.sum(self.r_meas_divisor) else 0 - - def get_r_split(self): - try: - r_split_bin = (1/math.sqrt(2))*(flex.sum(flex.abs(self.I_even - self.I_odd))/(flex.sum(self.I_even + self.I_odd)*0.5)) - except Exception as e: - print("Warning: R_split calculation failed.") - print(e) - r_split_bin =0 - return r_split_bin if self.get_size() else 0 - - def get_cc12(self): - cc12 = flex.linear_correlation(self.I_even, self.I_odd).coefficient() - return (cc12, self.get_size()) if self.get_size() else (0,0) - - def get_cciso(self, miller_array_iso): - cciso, n_refl_cciso = (0,0) - if miller_array_iso: - matches_iso = miller.match_multi_indices( - miller_indices_unique=miller_array_iso.indices(), - miller_indices=self.miller_array_merge.indices()) - I_iso = flex.double([miller_array_iso.data()[pair[0]] for pair in matches_iso.pairs()]) - I_merge_match_iso = flex.double([self.I_merge[pair[1]] for pair in matches_iso.pairs()]) - n_refl_cciso = len(matches_iso.pairs()) - if len(matches_iso.pairs()) > 0 : cciso = flex.linear_correlation(I_merge_match_iso, I_iso).coefficient() - return cciso, n_refl_cciso - - def get_mean_IoversigI(self): - return flex.mean(self.I_merge/self.sigI_merge) if self.get_size() else 0 - - def get_mean_I(self): - return flex.mean(self.I_merge) if self.get_size() else 0 - - def get_mean_sigI(self): - return flex.mean(self.sigI_merge) if self.get_size() else 0 - - def get_second_moment(self): - second_moment = 0 - if self.get_size(): - i_acentric = (self.miller_array_merge.centric_flags().data() == False) - I_acentric = self.I_merge.select(i_acentric) - second_moment = flex.mean(I_acentric**2)/(flex.mean(I_acentric)**2) - return second_moment - - def reduce_by_selection(self, selections): - """from selections (flex.bool), filter all class attributes""" - self.miller_indices_merge = self.miller_indices_merge.select(selections) - self.I_merge = self.I_merge.select(selections) - self.sigI_merge = self.sigI_merge.select(selections) - self.r_meas_div = self.r_meas_div.select(selections) - self.r_meas_divisor = self.r_meas_divisor.select(selections) - self.multiplicities = self.multiplicities.select(selections) - self.I_even = self.I_even.select(selections) - self.I_odd = self.I_odd.select(selections) - self.I_even_h = self.I_even_h.select(selections) - self.I_odd_h = self.I_odd_h.select(selections) - self.I_even_k = self.I_even_k.select(selections) - self.I_odd_k = self.I_odd_k.select(selections) - self.I_even_l = self.I_even_l.select(selections) - self.I_odd_l = self.I_odd_l.select(selections) - if self.miller_array_merge: - self.miller_array_merge = self.miller_array_merge.select(selections) - - def reduce_by_miller_index(self, miller_indices): - sequences = list(range(self.get_size())) - matches = miller.match_multi_indices( - miller_indices_unique=miller_indices, - miller_indices=self.miller_indices_merge) - pair_1 = flex.int([pair[1] for pair in matches.pairs()]) - sequences_bin = flex.size_t([sequences[pair_1[j]] for j in range(len(matches.pairs()))]) - self.reduce_by_selection(sequences_bin) - - def reduce_to_cone_on_axis(self, axis_point_2, fraction_percent): - miller_array_reduced = self.miller_array_merge.remove_cone(fraction_percent, axis_point_2=axis_point_2, negate=True) - self.reduce_by_miller_index(miller_array_reduced.indices()) diff --git a/prime/postrefine/mod_misc.py b/prime/postrefine/mod_misc.py deleted file mode 100644 index 76230f1769c..00000000000 --- a/prime/postrefine/mod_misc.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import absolute_import, division, print_function -from six.moves import range - -class misc_handler(object): - def calculate_SE(self, results, iparams): - """ - Take all post-refinement results, calculate covariances and new SE - """ - if results[0].grad_set is None: - return results - else: - #get miller array iso for comparison, if given - mxh = mx_handler() - flag_hklisoin_found, miller_array_iso = mxh.get_miller_array_from_reflection_file(iparams.hklisoin) - #get reference set - import os - fileseq_list = flex.int() - for file_in in os.listdir(iparams.run_no): - if file_in.endswith('.mtz'): - file_split = file_in.split('_') - if len(file_split) > 3: - fileseq_list.append(int(file_split[2])) - if len(fileseq_list) == 0: - hklref = 'mean_scaled_merge.mtz' - else: - hklref = 'postref_cycle_'+str(flex.max(fileseq_list))+'_merge.mtz' - flag_hklref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(iparams.run_no+'/'+hklref) - #calculate covariance - X = np.array([[pres.G, pres.B, pres.tau, pres.rotx, pres.roty, pres.ry, pres.rz, pres.r0, pres.re, \ - pres.uc_params[0], pres.uc_params[1],pres.uc_params[2],pres.uc_params[3], \ - pres.uc_params[4],pres.uc_params[5]] for pres in results]).T - COV = np.cov(X) - COV_diag = flex.double([i for i in COV.diagonal()]) - #calculate standard errors - output_results = [] - for pres in results: - sigI = pres.observations.sigmas() - observations_old = pres.observations.deep_copy() - var_set = (pres.grad_set**2) * COV_diag - sin_theta_over_lambda_sq = pres.observations.two_theta(wavelength=pres.wavelength).\ - sin_theta_over_lambda_sq().data() - d_spacings = pres.observations.d_spacings().data() - scale_factors_by_indices = pres.G * flex.exp(-2*pres.B*sin_theta_over_lambda_sq) - var_scale_factors = var_set[0] + var_set[1] - var_partiality = flex.sum(var_set[3:]) - err_scale_factors = var_scale_factors/(scale_factors_by_indices**2) - err_partiality = var_partiality/(pres.partiality**2) - err_tau = var_set[2]/((pres.tau**2)+1E-9) - #determine weight - I_o_full = ((4 * pres.rs_set * pres.observations.data())/(3 * pres.e_width_set * scale_factors_by_indices * pres.partiality)) + pres.tau - observations_full = pres.observations.customized_copy(data=I_o_full) - observations_err_scale = pres.observations.customized_copy(data=err_scale_factors) - observations_err_p = pres.observations.customized_copy(data=err_partiality) - observations_err_tau = pres.observations.customized_copy(data=flex.double([err_tau]*len(pres.observations.data()))) - flag_reset_weight = False - if flag_hklref_found: - ma_ref, ma_obs_full = miller_array_ref.common_sets(observations_full, assert_is_similar_symmetry=False) - dummy, ma_err_scale = miller_array_ref.common_sets(observations_err_scale, assert_is_similar_symmetry=False) - dummy, ma_err_p = miller_array_ref.common_sets(observations_err_p, assert_is_similar_symmetry=False) - dummy, ma_err_tau = miller_array_ref.common_sets(observations_err_tau, assert_is_similar_symmetry=False) - r1_factor = ma_ref.data()-ma_obs_full.data() - SE_I_WEIGHT = max([flex.linear_correlation((ma_obs_full.sigmas()**2), flex.log(r1_factor**2)).coefficient(), 0]) - SE_SCALE_WEIGHT = max([flex.linear_correlation(ma_err_scale.data(), flex.log(r1_factor**2)).coefficient(), 0]) - SE_EOC_WEIGHT = max([flex.linear_correlation(ma_err_p.data(), flex.log(r1_factor**2)).coefficient(), 0]) - SE_TAU_WEIGHT = max([flex.linear_correlation(ma_err_tau.data(), flex.log(r1_factor**2)).coefficient(), 0]) - else: - flag_reset_weight = True - if True: - SE_I_WEIGHT = self.CONST_SE_I_WEIGHT - SE_SCALE_WEIGHT = self.CONST_SE_SCALE_WEIGHT - SE_EOC_WEIGHT = self.CONST_SE_EOC_WEIGHT - SE_TAU_WEIGHT = self.CONST_SE_TAU_WEIGHT - #calculate new sigma - new_sigI = flex.sqrt((SE_I_WEIGHT*(sigI**2)) + \ - (SE_SCALE_WEIGHT*(err_scale_factors * (flex.sum(sigI**2)/(flex.sum(err_scale_factors)+1E-9)))) + \ - (SE_TAU_WEIGHT*(err_tau * (flex.sum(sigI**2)/((err_tau*len(sigI))+1E-9)))) + \ - (SE_EOC_WEIGHT*(err_partiality * (flex.sum(sigI**2)/(flex.sum(err_partiality)+1E-9))))) - #for i in new_sigI: - # if math.isnan(i) or i == float('inf') or i<0.1: - # print i, SE_I_WEIGHT, SE_SCALE_WEIGHT, SE_EOC_WEIGHT, SE_TAU_WEIGHT - pres.set_params(observations=pres.observations.customized_copy(sigmas=new_sigI), - observations_original=pres.observations_original.customized_copy(sigmas=new_sigI)) - output_results.append(pres) - #for plotting - if iparams.flag_plot_expert: - plt.subplot(521) - plt.scatter(1/(d_spacings**2), scale_factors_by_indices, s=10, marker='x', c='r') - plt.title('Scale factors') - plt.subplot(522) - plt.scatter(1/(d_spacings**2), pres.partiality, s=10, marker='x', c='r') - plt.title('Partiality') - plt.subplot(523) - plt.scatter(1/(d_spacings**2), err_scale_factors, s=10, marker='x', c='r') - plt.title('Error in scale factors') - plt.subplot(524) - plt.scatter(1/(d_spacings**2), err_partiality, s=10, marker='x', c='r') - plt.title('Error in partiality') - plt.subplot(525) - plt.scatter(1/(d_spacings**2), sigI, s=10, marker='x', c='r') - plt.title('Original sigmas') - plt.subplot(526) - plt.scatter(1/(d_spacings**2), new_sigI, s=10, marker='x', c='r') - plt.title('New sigmas') - plt.subplot(527) - plt.scatter(1/(d_spacings**2), flex.log(pres.observations.data()), s=10, marker='x', c='r') - plt.title('Original I') - plt.subplot(528) - plt.scatter(1/(d_spacings**2), flex.log(observations_full.data()), s=10, marker='x', c='r') - plt.title('New I') - if miller_array_iso is not None: - ma_iso, ma_obs_old = miller_array_iso.common_sets(observations_old, assert_is_similar_symmetry=False) - ma_iso, ma_obs_full = miller_array_iso.common_sets(observations_full, assert_is_similar_symmetry=False) - plt.subplot(529) - plt.scatter(ma_obs_old.sigmas(), flex.log(flex.abs(ma_iso.data()-ma_obs_old.data())), s=10, marker='x', c='r') - plt.title('Original SE vs Log Residual (R=%6.1f CC=%6.2f)'%(flex.sum(flex.abs(ma_iso.data()-ma_obs_old.data()))/flex.sum(flex.abs(ma_obs_old.data())), \ - flex.linear_correlation(ma_obs_old.sigmas(), flex.log(flex.abs(ma_iso.data()-ma_obs_old.data()))).coefficient())) - plt.subplot(5,2,10) - plt.scatter(ma_obs_full.sigmas(), flex.log(flex.abs(ma_iso.data()-ma_obs_full.data())), s=10, marker='x', c='r') - plt.title('New SE vs Log Residual (R=%6.1f CC=%6.2f)'%(flex.sum(flex.abs(ma_iso.data()-ma_obs_full.data()))/flex.sum(flex.abs(ma_obs_full.data())), \ - flex.linear_correlation(ma_obs_full.sigmas(), flex.log(flex.abs(ma_iso.data()-ma_obs_full.data()))).coefficient())) - plt.show() - return output_results - -class basis_handler(object): - ''' - classdocs - ''' - - def __init__(self): - ''' - Constructor - ''' - - def calc_direct_space_matrix(self, my_unit_cell, rotation_matrix): - - #calculate the conversion matrix (from fractional to cartesian coordinates - frac2cart_matrix = my_unit_cell.orthogonalization_matrix() - from scitbx.matrix import sqr - frac2cart_matrix = sqr(frac2cart_matrix) - - #calculate direct_space matrix - direct_space_matrix = frac2cart_matrix.transpose()*rotation_matrix - - return direct_space_matrix - - -class svd_handler(object): - ''' - Singular value decomposion - Solve linear equations with best fit basis - ''' - # Input: expects Nx3 matrix of points - # Returns R,t - # R = 3x3 rotation matrix - # t = 3x1 column vector - def __init__(self): - """ - Constructor - """ - - def rigid_transform_3D(self, A, B): - assert len(A) == len(B) - N = A.shape[0]; # total points - centroid_A = np.mean(A, axis=0) - centroid_B = np.mean(B, axis=0) - # centre the points - AA = A - np.tile(centroid_A, (N, 1)) - BB = B - np.tile(centroid_B, (N, 1)) - # dot is matrix multiplication for array - H = np.transpose(AA) * BB - U, S, Vt = np.linalg.svd(H) - R = Vt.T * U.T - # special reflection case - if np.linalg.det(R) < 0: - #print "Reflection detected" - Vt[2,:] *= -1 - R = Vt.T * U.T - t = -R*centroid_A.T + centroid_B.T - return R, t - -class wilson_plot_handler(object): - """ - Take miller array and show Wilson Plot - """ - def __init__(self): - """ - Constructor - """ - - def show_plot(self, miller_array_in, n_bins=None): - try: - import matplotlib.pyplot as plt - except Exception as e: - print("Warning: error importing matplotlib.pyplot") - print(e) - return - if n_bins is None: - binner = miller_array_in.setup_binner(auto_binning=True) - else: - binner = miller_array_in.setup_binner(n_bins=n_bins) - binner_indices = binner.bin_indices() - avg_I_bin = flex.double() - one_dsqr_bin = flex.double() - for i in range(1, n_bins+1): - i_binner = (binner_indices == i) - I_sel = observations_original.data().select(i_binner) - avg_I_bin.append(np.mean(I_sel)) - one_dsqr_bin.append(1/binner.bin_d_range(i)[1]**2) - x_axis = one_dsqr_bin - fig, ax1 = plt.subplots() - ln1 = ax1.plot(x_axis, avg_I_bin, linestyle='-', linewidth=2.0, c='b') - ax1.set_xlabel('1/d^2') - ax1.set_ylabel('', color='b') - plt.grid() - plt.show() diff --git a/prime/postrefine/mod_mx.py b/prime/postrefine/mod_mx.py deleted file mode 100644 index 8153a646a45..00000000000 --- a/prime/postrefine/mod_mx.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import absolute_import, division, print_function -from iotbx import reflection_file_reader - -class mx_handler(object): - """ - Author : Uervirojnangkoorn, M. - Created : 4/15/2016 - A collection of macromolecular-crystallagphic wrapper functions - """ - def __init__(self): - """ - Constructor - """ - - def get_asu_contents(self, n_residues): - asu_contents = None - if n_residues > 0: - asu_contents = {"H":8.0*float(n_residues), - "C":5.0*float(n_residues), - "N":1.5*float(n_residues), - "O":1.2*float(n_residues) - } - return asu_contents - - def get_miller_array_from_reflection_file(self, hklisoin): - #get iso if given - flag_hklisoin_found = False - miller_array_iso = None - if hklisoin is not None: - flag_hklisoin_found = True - reflection_file_iso = reflection_file_reader.any_reflection_file(hklisoin) - miller_arrays_iso=reflection_file_iso.as_miller_arrays() - is_found_iso_as_intensity_array = False - is_found_iso_as_amplitude_array = False - for miller_array in miller_arrays_iso: - if miller_array.is_xray_intensity_array(): - miller_array_iso = miller_array.deep_copy() - is_found_iso_as_intensity_array = True - break - elif miller_array.is_xray_amplitude_array(): - is_found_iso_as_amplitude_array = True - miller_array_converted_to_intensity = miller_array.as_intensity_array() - if is_found_iso_as_intensity_array == False: - if is_found_iso_as_amplitude_array: - miller_array_iso = miller_array_converted_to_intensity.deep_copy() - else: - flag_hklisoin_found = False - return flag_hklisoin_found, miller_array_iso - - def get_resolution_step_for_B(self, iparams): - resolution_gap = 7 - iparams.scale.d_min - resolution_step = resolution_gap/ iparams.n_bins - return resolution_step diff --git a/prime/postrefine/mod_outlier.py b/prime/postrefine/mod_outlier.py deleted file mode 100644 index 2b22a21a6f7..00000000000 --- a/prime/postrefine/mod_outlier.py +++ /dev/null @@ -1,58 +0,0 @@ -''' -Author : Uervirojnangkoorn, M. -Created : 7/13/2014 -Detecting outliers with Wilson statistics (Read, 1999) -use the normalized intensity (E) limits (acentric 3.72, centric 4.89) -''' -from __future__ import absolute_import, division, print_function -from cctbx.array_family import flex -from cctbx import statistics -from six.moves import range - -class outlier_handler(object): - ''' - A wrapper class for finding outliers - ''' - - def __init__(self): - ''' - Constructure - ''' - - def good_i_flags(self, miller_array_i, iparams, flag_show_summary=False): - miller_array_f = miller_array_i.f_sq_as_f() - good_i_flags = flex.bool([True]*len(miller_array_f.data())) - - try: - miller_array_f.setup_binner(auto_binning=True) - miller_array_e = miller_array_f.quasi_normalize_structure_factors() - i_seq = flex.int([i for i in range(len(miller_array_e.data()))]) - - e_lim_acentric = 3.72 - e_lim_centric = 4.89 - centric_flags_data = miller_array_e.centric_flags().data() - data_e_acentric = miller_array_e.data().select(centric_flags_data==False) - i_seq_acentric = i_seq.select(centric_flags_data==False) - data_e_centric = miller_array_e.data().select(centric_flags_data==True) - i_seq_centric = i_seq.select(centric_flags_data==True) - - i_seq_acentric_outliers = i_seq_acentric.select(data_e_acentric>=e_lim_acentric) - i_seq_centric_outliers = i_seq_centric.select(data_e_centric>=e_lim_centric) - - i_seq_outliers = sorted(i_seq_acentric_outliers.concatenate(i_seq_centric_outliers)) - - for i in i_seq_outliers: - good_i_flags[i] = False - - if flag_show_summary: - print('Acentric outliers:') - for i in i_seq_acentric_outliers: - print(miller_array_e.indices()[i], miller_array_e.data()[i]) - - print('Centric outliers:') - for i in i_seq_centric_outliers: - print(miller_array_e.indices()[i], miller_array_e.data()[i]) - except Exception: - dummy = 0 - - return good_i_flags diff --git a/prime/postrefine/mod_partiality.py b/prime/postrefine/mod_partiality.py deleted file mode 100644 index cbce4b34d0d..00000000000 --- a/prime/postrefine/mod_partiality.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import absolute_import, division, print_function -from cctbx.array_family import flex -from scitbx.matrix import sqr, col -from cctbx.crystal_orientation import crystal_orientation, basis_type -import math -import numpy as np - -class partiality_handler(object): - """ - mod_partiality: - 1. Calculate partiality for given - miller indices, crystal orientation, unit cell, wavelength. - 2. Cacluate spot centroid delta distance - """ - def __init__(self): - """ - Intialitze parameters - """ - - def calc_full_refl(self, I_o_p_set, sin_theta_over_lambda_sq_set, - G, B, p_set, rs_set, flag_volume_correction=True): - I_o_full_set = I_o_p_set/(G * flex.exp(-2*B*sin_theta_over_lambda_sq_set) * p_set) - return I_o_full_set - - def calc_spot_radius(self, a_star_matrix, miller_indices, wavelength): - #calculate spot_radius based on rms delta_S for all spots - S0 = -1*col((0,0,1./wavelength)) - sd_array = a_star_matrix.elems * miller_indices.as_vec3_double() + S0.elems - rh_set = sd_array.norms() - (1/wavelength) - return rh_set.standard_deviation_of_the_sample() - - def voigt(self, x, sig, nu): - if nu < 0: - nu = 0 - elif nu > 1: - nu = 1 - f1 = nu * math.sqrt(math.log(2)/math.pi) * flex.exp(-4*math.log(2)*((x/sig)**2)) * (1/abs(sig)) - f2 = (1-nu)/(math.pi*abs(sig)*(1+(4*((x/sig)**2)))) - f3 = ((nu * math.sqrt(math.log(2)/math.pi))/abs(sig)) + ((1-nu)/(math.pi*abs(sig))) - svx = (f1 + f2)/f3 - return svx - - def lognpdf(self, x, FWHM, zero): - #find sig from root of this function - zero = np.abs(zero) - sig_range = np.arange(50)/100 - t = sig_range * math.sqrt(math.log(4)) - sig_set = np.array([sig_range[np.argmin(np.abs(( fwhm - (zero * (np.exp(t) - np.exp(-1*t))) )))] for fwhm in FWHM]) - #calc x0 - x0 = math.log(zero) + sig_set**2 - g = 1/( sig_set * math.sqrt(2*math.pi) * np.exp(x0-((sig_set**2)/2)) ) - #calc lognpdf - X = zero - x - f1 = 1/( X * sig_set * math.sqrt(2*math.pi) ) - f2 = np.exp( -1 * (np.log(X)-x0)**2 / (2*(sig_set**2)) ) - svx = flex.double(f1 * f2 / g) - return svx - - def calc_partiality_anisotropy_set(self, my_uc, rotx, roty, miller_indices, - ry, rz, r0, re, nu, - bragg_angle_set, alpha_angle_set, wavelength, crystal_init_orientation, - spot_pred_x_mm_set, spot_pred_y_mm_set, detector_distance_mm, - partiality_model, flag_beam_divergence): - #use III.4 in Winkler et al 1979 (A35; P901) for set of miller indices - O = sqr(my_uc.orthogonalization_matrix()).transpose() - R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() - CO = crystal_orientation(O*R, basis_type.direct) - CO_rotate = CO.rotate_thru((1,0,0), rotx - ).rotate_thru((0,1,0), roty) - A_star = sqr(CO_rotate.reciprocal_matrix()) - S0 = -1*col((0,0,1./wavelength)) - #caculate rs - rs_set = r0 + (re * flex.tan(bragg_angle_set)) - if flag_beam_divergence: - rs_set += ((ry * flex.cos(alpha_angle_set))**2 + (rz * flex.sin(alpha_angle_set))**2)**(1/2) - #calculate rh - x = A_star.elems * miller_indices.as_vec3_double() - sd_array = x + S0.elems - rh_set = sd_array.norms() - (1/wavelength) - #calculate partiality - if partiality_model == "Lorentzian": - partiality_set = ((rs_set**2)/((2*(rh_set**2))+(rs_set**2))) - elif partiality_model == "Voigt": - partiality_set = self.voigt(rh_set, rs_set, nu) - elif partiality_model == "Lognormal": - partiality_set = self.lognpdf(rh_set, rs_set, nu) - #calculate delta_xy - if sum(spot_pred_y_mm_set) == 0: - #hack for dials integration - spot_pred_x_mm_set is s1 * to be fixed * - delta_xy_set = (spot_pred_x_mm_set - sd_array).norms() - else: - d_ratio = -detector_distance_mm/sd_array.parts()[2] - calc_xy_array = flex.vec3_double(sd_array.parts()[0]*d_ratio, \ - sd_array.parts()[1]*d_ratio, flex.double([0]*len(d_ratio))) - pred_xy_array = flex.vec3_double(spot_pred_x_mm_set, spot_pred_y_mm_set, flex.double([0]*len(d_ratio))) - delta_xy_set = (pred_xy_array - calc_xy_array).norms() - return partiality_set, delta_xy_set, rs_set, rh_set diff --git a/prime/postrefine/mod_pdb.py b/prime/postrefine/mod_pdb.py deleted file mode 100644 index 14e86863595..00000000000 --- a/prime/postrefine/mod_pdb.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import absolute_import, division, print_function - -class pdb_handler(object): - ''' - Author : Uervirojnangkoorn, M. - Created : 7/13/2014 - Determines number of atoms in a given pdb or sequence file. - ''' - - def __init__(self, file_name_pdb): - ''' - Constructor - ''' - file_pdb = open(file_name_pdb,'r') - self.data_pdb = file_pdb.read().split("\n") - - - def print_file_content(self): - - for line in self.data_pdb: - print(line) - - def atom_stats(self): - - cn_C = 0 - cn_H = 0 - cn_N = 0 - cn_O = 0 - cn_S = 0 - - - for line in self.data_pdb: - if line.find('ATOM') == 0 or line.find('HETATM') == 0: - col = line.split(' ') - atom = col[len(col)-3].strip() - if atom == 'N': - cn_N += 1 - elif atom == 'O': - cn_O += 1 - elif atom == 'C': - cn_C += 1 - elif atom == 'S': - cn_S += 1 - elif atom == 'H': - cn_H += 1 - - - #print 'C ', cn_C, 'H', cn_H, 'N', cn_N, 'O', cn_O, 'S', cn_S - - return (cn_C, cn_H, cn_N, cn_O, cn_S) diff --git a/prime/postrefine/mod_perf.py b/prime/postrefine/mod_perf.py deleted file mode 100644 index d1b48054bed..00000000000 --- a/prime/postrefine/mod_perf.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import absolute_import, division, print_function - -from datetime import datetime - -class perf_handler(): - - def __init__(self, module_name): - self.module_name = module_name - self.t_st = datetime.now() - - def get_elapsed_times(self): - self.t_en = datetime.now() - self.t_spent = self.t_en - self.t_st - print(self.module_name, self.t_st.strftime("%d/%m/%Y %H:%M:%S"), self.t_en.strftime("%d/%m/%Y %H:%M:%S"), self.t_spent.microseconds) diff --git a/prime/postrefine/mod_plotter.py b/prime/postrefine/mod_plotter.py deleted file mode 100644 index 3c2365fe39d..00000000000 --- a/prime/postrefine/mod_plotter.py +++ /dev/null @@ -1,202 +0,0 @@ -from __future__ import division, unicode_literals, print_function,\ - absolute_import - -''' -Author : Lyubimov, A.Y. -Created : 05/25/2016 -Last Changed: 07/17/2019 -Description : PRIME Result Plotter module -''' - -import numpy as np - -from matplotlib import gridspec, rc - -from iota.gui.plotter import Plotter as IOTAPlotter - -class Plotter(IOTAPlotter): - ''' Class with function to plot various PRIME charts (includes Table 1) ''' - - def __init__(self, parent, info, output_dir=None, anomalous_flag=False, - *args, **kwargs): - super(Plotter, self).__init__(parent=parent, info=info, *args, **kwargs) - self.target_anomalous_flag = anomalous_flag - self.output_dir = output_dir - - def table_one(self, as_tex=False): - ''' Constructs Table 1 for GUI or logging ''' - - if as_tex: - A = r'$\AA$' - d = r'$\circ$' - a = r'$\alpha$' - b = r'$\beta$' - g = r'$\gamma$' - s = r'$\sigma$' - h = r'$\frac{1}{2}$' - rm = r'$\_merge$' - else: - A = '\N{ANGSTROM SIGN}' - d = '\N{DEGREE SIGN}' - a = '\N{GREEK SMALL LETTER ALPHA}' - b = '\N{GREEK SMALL LETTER BETA}' - g = '\N{GREEK SMALL LETTER GAMMA}' - s = '\N{GREEK SMALL LETTER SIGMA}' - h = '\u00BD' - rm = '_merge' - t1_rlabels = ['No. of accepted images', - 'No. of rejected images', - 'Space Group', - 'Cell dimensions', - ' a, b, c ({}) '.format(A), - ' {}, {}, {} ({}) '.format(a, b, g, d), - 'Resolution ({}) '.format(A), - 'Completeness (%)', - 'Multiplicity', - 'I / {}(I) '.format(s), - 'CC{} '.format(h), - 'R{}'.format(rm)] - - uc_edges = '{:4.2f}, {:4.2f}, {:4.2f}'.format(self.info['mean_a'][-1], - self.info['mean_b'][-1], - self.info['mean_c'][-1]) - uc_angles = '{:4.2f}, {:4.2f}, {:4.2f}'.format(self.info['mean_alpha'][-1], - self.info['mean_beta'][-1], - self.info['mean_gamma'][-1]) - res_total = '{:4.2f} - {:4.2f}'.format(self.info['total_res_max'][-1], - self.info['total_res_min'][-1]) - res_last_shell = '{:4.2f} - {:4.2f}' \ - ''.format(self.info['binned_resolution'][-1][-2], - self.info['binned_resolution'][-1][-1]) - - n_frames_bad = self.info['n_frames_bad_cc'][-1] + \ - self.info['n_frames_bad_G'][-1] + \ - self.info['n_frames_bad_uc'][-1] + \ - self.info['n_frames_bad_gamma_e'][-1] + \ - self.info['n_frames_bad_SE'][-1] - t1_data = [['{}'.format(self.info['n_frames_good'][-1])], - ['{}'.format(n_frames_bad)], - ['{}'.format(self.info['space_group_info'][-1])], - [''], - ['{}'.format(uc_edges)], - ['{}'.format(uc_angles)], - ['{} ({})'.format(res_total, res_last_shell)], - ['{:4.2f} ({:4.2f})'.format(self.info['total_completeness'][-1], - self.info['binned_completeness'][-1][-1])], - ['{:4.2f} ({:4.2f})'.format(self.info['total_n_obs'][-1], - self.info['binned_n_obs'][-1][-1])], - ['{:4.2f} ({:4.2f})'.format(self.info['total_i_o_sigi'][-1], - self.info['binned_i_o_sigi'][-1][-1])], - ['{:4.2f} ({:4.2f})'.format(self.info['total_cc12'][-1], - self.info['binned_cc12'][-1][-1]*100)], - ['{:4.2f} ({:4.2f})'.format(self.info['total_rmerge'][-1], - self.info['binned_rmerge'][-1][-1])] - ] - - return t1_rlabels, t1_data - - def flatten_table_one_data(self, as_tex=False): - rlabels, tb1_data = self.table_one(as_tex=as_tex) - - # Flatten data list of lists (works this once, since each sub-list - # contains a single item) - tb1_data = list(zip(*tb1_data))[0] - data = list(zip(rlabels, tb1_data)) - return data - - def table_one_text(self): - data = self.flatten_table_one_data() - return self.plot_table_text(data=data) - - def table_one_figure(self): - data = self.flatten_table_one_data() - self.plot_table(data=data) - - def stat_charts(self): - ''' Displays charts of CC1/2, Completeness, Multiplicity and I / sig(I) - per resolution bin after the final cycle of post-refinement ''' - - gsp = gridspec.GridSpec(2, 2) - gsp.update(wspace=0.1, hspace=0.1) - - self.figure.set_alpha(0) - rc('font', family='sans-serif', size=12) - rc('mathtext', default='regular') - - x = self.info['binned_resolution'][-1] - bins = np.arange(len(x)) - xlabels = ["{:.2f}".format(i) for i in x] - sel_bins = bins[0::len(bins) // 6] - sel_xlabels = [xlabels[t] for t in sel_bins] - - # Plot CC1/2 vs. resolution - ax_cc12 = self.figure.add_subplot(gsp[0]) - reslabel = 'Resolution ({})'.format(r'$\AA$') - ax_cc12.set_xlabel(reslabel, fontsize=15) - ax_cc12.ticklabel_format(axis='y', style='plain') - ax_cc12.set_ylim(0, 100) - - if self.target_anomalous_flag: - ax_cc12.set_ylabel(r'$CC_{1/2}$ anom (%)', fontsize=15) - else: - ax_cc12.set_ylabel(r'$CC_{1/2}$ (%)', fontsize=15) - ax_cc12.set_xticks(sel_bins) - ax_cc12.set_xticklabels(sel_xlabels) - ax_cc12.grid(True) - cc12_start_percent = [c * 100 for c in self.info['binned_cc12'][0]] - cc12_end_percent = [c * 100 for c in self.info['binned_cc12'][-1]] - start, = ax_cc12.plot(bins, cc12_start_percent, c='#7fcdbb', lw=2) - end, = ax_cc12.plot(bins, cc12_end_percent, c='#2c7fb8', lw=3) - labels = ['Initial', 'Final'] - ax_cc12.legend([start, end], labels, loc='lower left', - fontsize=9, fancybox=True) - - # Plot Completeness vs. resolution - ax_comp = self.figure.add_subplot(gsp[1]) - ax_comp.set_xlabel(reslabel, fontsize=15) - ax_comp.ticklabel_format(axis='y', style='plain') - ax_comp.set_ylabel('Completeness (%)', fontsize=15) - ax_comp.set_xticks(sel_bins) - ax_comp.set_xticklabels(sel_xlabels) - ax_comp.set_ylim(0, 100) - ax_comp.grid(True) - start, = ax_comp.plot(bins, self.info['binned_completeness'][0], - c='#7fcdbb', lw=2) - end, = ax_comp.plot(bins, self.info['binned_completeness'][-1], c='#2c7fb8', - lw=3) - labels = ['Initial', 'Final'] - ax_comp.legend([start, end], labels, loc='lower left', - fontsize=9, fancybox=True) - - # Plot Multiplicity (no. of observations) vs. resolution - ax_mult = self.figure.add_subplot(gsp[2]) - ax_mult.set_xlabel(reslabel, fontsize=15) - ax_mult.ticklabel_format(axis='y', style='plain') - ax_mult.set_ylabel('# of Observations', fontsize=15) - ax_mult.set_xticks(sel_bins) - ax_mult.set_xticklabels(sel_xlabels) - ax_mult.grid(True) - start, = ax_mult.plot(bins, self.info['binned_n_obs'][0], c='#7fcdbb', lw=2) - end, = ax_mult.plot(bins, self.info['binned_n_obs'][-1], c='#2c7fb8', lw=3) - labels = ['Initial', 'Final'] - ax_mult.legend([start, end], labels, loc='lower left', - fontsize=9, fancybox=True) - - # Plot I / sig(I) vs. resolution - ax_i_sigi = self.figure.add_subplot(gsp[3]) - ax_i_sigi.set_xlabel(reslabel, fontsize=15) - ax_i_sigi.ticklabel_format(axis='y', style='plain') - ax_i_sigi.set_ylabel(r'I / $\sigma$(I)', fontsize=15) - ax_i_sigi.set_xticks(sel_bins) - ax_i_sigi.set_xticklabels(sel_xlabels) - ax_i_sigi.grid(True) - start, = ax_i_sigi.plot(bins, self.info['binned_i_o_sigi'][0], c='#7fcdbb', - lw=2) - end, = ax_i_sigi.plot(bins, self.info['binned_i_o_sigi'][-1], c='#2c7fb8', - lw=3) - labels = ['Initial', 'Final'] - ax_i_sigi.legend([start, end], labels, loc='lower left', - fontsize=9, fancybox=True) - - # self.figure.set_tight_layout(True) - self.draw(tight_layout=False) diff --git a/prime/postrefine/mod_results.py b/prime/postrefine/mod_results.py deleted file mode 100644 index a837972a023..00000000000 --- a/prime/postrefine/mod_results.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import absolute_import, division, print_function -from cctbx.array_family import flex -from cctbx.uctbx import unit_cell - -class postref_results(object): - ''' - Author : Uervirojnangkoorn, M. - Created : 7/13/2014 - A wrapper class to store refinement result - ''' - - def __init__(self): - ''' - Do nothing - ''' - - def set_params(self, observations=None, - observations_original=None, - refined_params=None, - stats=None, - partiality=None, - rs_set=None, - rh_set=None, - frame_no=None, - pickle_filename=None, - wavelength=None, - crystal_orientation=None, - detector_distance_mm=None): - - self.observations = observations - self.observations_original = observations_original - self.refined_params = refined_params - self.partiality = partiality - self.rs_set = rs_set - self.rh_set = rh_set - self.frame_no = frame_no - self.pickle_filename = pickle_filename - self.wavelength = wavelength - self.detector_distance_mm = detector_distance_mm - - #refined_params - #note params = G,B,rotx,roty,ry,a,b,c,alpha,beta,gamma - self.G = refined_params[0] - self.B = refined_params[1] - self.rotx = refined_params[2] - self.roty = refined_params[3] - self.ry = refined_params[4] - self.rz = refined_params[5] - self.r0 = refined_params[6] - self.re = refined_params[7] - self.voigt_nu = refined_params[8] - self.uc_params = flex.double([refined_params[9], refined_params[10], refined_params[11], - refined_params[12], refined_params[13], refined_params[14]]) - self.unit_cell = unit_cell((refined_params[9], refined_params[10], refined_params[11], - refined_params[12], refined_params[13], refined_params[14])) - self.crystal_orientation = crystal_orientation - - #SE, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final - self.SE = stats[0] - self.R_sq = stats[1] - self.CC_init = stats[2] - self.CC_final = stats[3] - self.R_init = stats[4] - self.R_final = stats[5] - self.R_xy_init = stats[6] - self.R_xy_final = stats[7] - self.CC_iso_init = stats[8] - self.CC_iso_final = stats[9] - - def get_full_observations(self): - sin_theta_over_lambda_sq = self.observations.two_theta(wavelength=self.wavelength).sin_theta_over_lambda_sq().data() - return self.observations.customized_copy( \ - data=self.observations.data()/(self.G * flex.exp(-2*self.B*sin_theta_over_lambda_sq) * self.partiality), \ - sigmas=self.observations.sigmas()/(self.G * flex.exp(-2*self.B*sin_theta_over_lambda_sq) * self.partiality)) diff --git a/prime/postrefine/mod_run.py b/prime/postrefine/mod_run.py deleted file mode 100644 index 8aea891322a..00000000000 --- a/prime/postrefine/mod_run.py +++ /dev/null @@ -1,16 +0,0 @@ -""" handle queue usage """ -from __future__ import absolute_import, division, print_function -__author__ = 'Monarin Uervirojnangkoorn, monarin@gmail.com' - -import time, os - -class run_handler(object): - - def check_done(self, iparams): - #only allow the module to continue if all frames are found or time out. - program_starts = time.time() - while((time.time() - program_starts < iparams.timeout_seconds) and \ - not os.path.isfile(os.path.join(iparams.run_no,'.done'))): - print("Running - Elapsed times: {0:6.1f} seconds".format(time.time() - program_starts)) - time.sleep(5) - print("Done. You can view your results and log file in ", iparams.run_no) diff --git a/prime/postrefine/mod_threads.py b/prime/postrefine/mod_threads.py deleted file mode 100644 index f6b32b35910..00000000000 --- a/prime/postrefine/mod_threads.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import division, print_function, absolute_import - -''' -Author : Lyubimov, A.Y. -Created : 05/01/2016 -Last Changed: 06/20/2019 -Description : PRIME GUI Threading module -''' - -import os -import wx -from threading import Thread - -from iota.threads.iota_threads import CustomRun - -# Platform-specific stuff -# TODO: Will need to test this on Windows at some point -if wx.Platform == '__WXGTK__': - norm_font_size = 10 - button_font_size = 12 - LABEL_SIZE = 14 - CAPTION_SIZE = 12 - python = 'python' -elif wx.Platform == '__WXMAC__': - norm_font_size = 12 - button_font_size = 14 - LABEL_SIZE = 14 - CAPTION_SIZE = 12 - python = "Python" -elif (wx.Platform == '__WXMSW__'): - norm_font_size = 9 - button_font_size = 11 - LABEL_SIZE = 11 - CAPTION_SIZE = 9 - -def str_split(string, delimiters=(' ', ','), maxsplit=0): - import re - rexp = '|'.join(map(re.escape, delimiters)) - return re.split(rexp, string, maxsplit) - - -# -------------------------------- Threading --------------------------------- # - -# Set up events for finishing one cycle and for finishing all cycles -tp_EVT_ALLDONE = wx.NewEventType() -EVT_ALLDONE = wx.PyEventBinder(tp_EVT_ALLDONE, 1) - -class AllDone(wx.PyCommandEvent): - ''' Send event when finished all cycles ''' - def __init__(self, etype, eid): - wx.PyCommandEvent.__init__(self, etype, eid) - -class PRIMEThread(Thread): - ''' Worker thread; generated so that the GUI does not lock up when - processing is running ''' - - def __init__(self, - parent, - prime_file, - out_file=None, - command=None, - cmd_args=None, - signal_finished=False, - debug=False, - verbose=False): - Thread.__init__(self) - self.parent = parent - self.prime_file = prime_file - self.out_file = out_file - self.command = command - self.cmd_args = cmd_args - self.signal_finished = signal_finished - self.verbose = verbose - self.debug = debug - self.job = None - - def run(self): - if os.path.isfile(self.out_file): - os.remove(self.out_file) - if self.command is None: - if self.cmd_args is None: - args = '' - else: - args = self.cmd_args - - cmd = 'prime.run {} {}'.format(self.prime_file, args) - else: - cmd = self.command - - if self.debug: - from libtbx import easy_run - if self.verbose: - print (cmd) - easy_run.fully_buffered(cmd, join_stdout_stderr=True).show_stdout() - else: - easy_run.fully_buffered(cmd, join_stdout_stderr=True) - else: - try: - self.job = CustomRun(command=cmd, join_stdout_stderr=True) - self.job.run() - except Exception as e: - print ("PRIME ERROR: ", e) - - if self.signal_finished: - evt = AllDone(tp_EVT_ALLDONE, -1) - wx.PostEvent(self.parent, evt) - - def abort(self): - # TODO: put in an LSF kill command - if self.job: - try: - self.job.kill_thread() - except Exception as e: - print ('PRIME THREAD ERROR: Cannot terminate thread! {}'.format(e)) diff --git a/prime/postrefine/mod_util.py b/prime/postrefine/mod_util.py deleted file mode 100644 index 1957a8e1565..00000000000 --- a/prime/postrefine/mod_util.py +++ /dev/null @@ -1,629 +0,0 @@ -from __future__ import absolute_import, division, print_function -from cctbx.uctbx import unit_cell -from cctbx import miller, crystal, statistics -from cctbx.array_family import flex -from iotbx import mtz -from libtbx.utils import Sorry -import math, os -import numpy as np -from copy import deepcopy -from six.moves import cPickle as pickle -from collections import Counter -from .mod_merge_data import merge_data_handler -from .mod_mx import mx_handler -from .mod_leastsqr import good_unit_cell -from six.moves import range - -class intensities_scaler(object): - """ - Author : Uervirojnangkoorn, M. - Created : 7/13/2014 - Merge equivalent reflections and report intensity and refinement statistics. - """ - def __init__(self): - """ - Constructor - """ - self.CONST_SE_MIN_WEIGHT = 0.17 - self.CONST_SE_MAX_WEIGHT = 1.0 - self.CONST_SIG_I_FACTOR = 1.5 - - def write_stat_pickle(self, iparams, stat_dict): - fname = iparams.run_no+'/stats/pickle_'+str(os.getpid())+'.stat' - if os.path.isfile(fname): - pickle_stat = pickle.load(open(fname,"rb")) - for key in stat_dict: - if key in pickle_stat: - pickle_stat[key].append(stat_dict[key][0]) - else: - pickle_stat[key] = stat_dict[key] - pickle.dump(pickle_stat, open(fname,"wb")) - else: - pickle.dump(stat_dict, open(fname,"wb")) - - def calc_avg_I_cpp(self, prep_output, iparams, avg_mode): - group_no, group_id_list, miller_index, miller_indices_ori, I, sigI, G, B, p_set, rs_set, wavelength_set, sin_theta_over_lambda_sq, SE, uc_mean, wavelength_mean, pickle_filename_set, txt_out = prep_output - from prime import Average_Mode, averaging_engine - if avg_mode == 'average': avg_mode_cpp = Average_Mode.Average - elif avg_mode == 'weighted': avg_mode_cpp = Average_Mode.Weighted - elif avg_mode == 'final': avg_mode_cpp = Average_Mode.Final - else: raise Sorry("Bad averaging mode selected: %s"%avg_mode) - sigma_max = iparams.sigma_rejection - engine = averaging_engine(group_no, group_id_list, miller_index, miller_indices_ori, I, sigI, G, B,p_set, rs_set, wavelength_set, sin_theta_over_lambda_sq, SE, pickle_filename_set) - engine.avg_mode = avg_mode_cpp - engine.sigma_max = sigma_max - engine.flag_volume_correction = iparams.flag_volume_correction - engine.n_rejection_cycle = iparams.n_rejection_cycle - engine.flag_output_verbose = iparams.flag_output_verbose - results = engine.calc_avg_I() - mdh = merge_data_handler() - mdh.extend_data(results.miller_index, results.I_avg, results.sigI_avg, (results.r_meas_top, results.r_meas_btm, results.multiplicity), (results.I_avg_even, results.I_avg_odd, results.I_avg_even_h, results.I_avg_odd_h, results.I_avg_even_k, results.I_avg_odd_k, results.I_avg_even_l, results.I_avg_odd_l), uc_mean, wavelength_mean) - return mdh, results.txt_obs_out, results.txt_reject_out - - def calc_mean_unit_cell(self, results): - uc_array = [list(pres.uc_params) for pres in results if pres is not None] - return np.mean(uc_array,0), np.median(uc_array,0), np.std(uc_array,0) - - def calc_mean_postref_parameters(self, results): - params_array = [[pres.G, pres.B, pres.ry, pres.rz, pres.re, pres.r0, \ - pres.voigt_nu, pres.rotx, pres.roty, pres.R_final, pres.R_xy_final, pres.SE] \ - for pres in results if (pres is not None and not math.isnan(pres.G) and not math.isnan(pres.B) \ - and not math.isnan(pres.ry) and not math.isnan(pres.rz) and not math.isnan(pres.re) and not math.isnan(pres.r0) \ - and not math.isnan(pres.voigt_nu) and not math.isnan(pres.rotx) and not math.isnan(pres.roty) \ - and not math.isnan(pres.R_final) and not math.isnan(pres.R_xy_final) and not math.isnan(pres.SE))] - return np.mean(params_array,0), np.median(params_array,0), np.std(params_array,0) - - def prepare_output(self, results, iparams, avg_mode): - if avg_mode == 'average': - cc_thres = 0 - else: - cc_thres = iparams.frame_accept_min_cc - std_filter = iparams.sigma_rejection - if iparams.flag_weak_anomalous: - if avg_mode == 'final': - target_anomalous_flag = iparams.target_anomalous_flag - else: - target_anomalous_flag = False - else: - target_anomalous_flag = iparams.target_anomalous_flag - pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(results) - G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean - G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med - G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std - #prepare data for merging - miller_indices_all = flex.miller_index() - miller_indices_ori_all = flex.miller_index() - I_all = flex.double() - sigI_all = flex.double() - G_all = flex.double() - B_all = flex.double() - p_all = flex.double() - rx_all = flex.double() - rs_all = flex.double() - rh_all = flex.double() - SE_all = flex.double() - sin_sq_all = flex.double() - wavelength_all = flex.double() - detector_distance_set = flex.double() - R_init_all = flex.double() - R_final_all = flex.double() - R_xy_init_all = flex.double() - R_xy_final_all = flex.double() - pickle_filename_all = flex.std_string() - filtered_results = [] - cn_good_frame, cn_bad_frame_SE, cn_bad_frame_uc, cn_bad_frame_cc, cn_bad_frame_G, cn_bad_frame_re = (0,0,0,0,0,0) - crystal_orientation_dict = {} - for pres in results: - if pres is not None: - pickle_filepath = pres.pickle_filename.split('/') - img_filename = pickle_filepath[len(pickle_filepath)-1] - flag_pres_ok = True - #check SE, CC, UC, G, B, gamma_e - if math.isnan(pres.G): - flag_pres_ok = False - if math.isnan(pres.SE) or np.isinf(pres.SE): - flag_pres_ok = False - if flag_pres_ok and SE_std > 0: - if abs(pres.SE-SE_med)/SE_std > std_filter: - flag_pres_ok = False - cn_bad_frame_SE += 1 - if flag_pres_ok and pres.CC_final < cc_thres: - flag_pres_ok = False - cn_bad_frame_cc += 1 - if flag_pres_ok: - if G_std > 0: - if abs(pres.G-G_med)/G_std > std_filter: - flag_pres_ok = False - cn_bad_frame_G += 1 - if flag_pres_ok: - if re_std > 0: - if abs(pres.re-re_med)/re_std > std_filter: - flag_pres_ok = False - cn_bad_frame_re += 1 - if flag_pres_ok and not good_unit_cell(pres.uc_params, iparams, iparams.merge.uc_tolerance): - flag_pres_ok = False - cn_bad_frame_uc += 1 - data_size = pres.observations.size() - if flag_pres_ok: - cn_good_frame += 1 - filtered_results.append(pres) - R_init_all.append(pres.R_init) - R_final_all.append(pres.R_final) - R_xy_init_all.append(pres.R_xy_init) - R_xy_final_all.append(pres.R_xy_final) - miller_indices_all.extend(pres.observations.indices()) - miller_indices_ori_all.extend(pres.observations_original.indices()) - I_all.extend(pres.observations.data()) - sigI_all.extend(pres.observations.sigmas()) - G_all.extend(flex.double([pres.G] * data_size)) - B_all.extend(flex.double([pres.B] * data_size)) - p_all.extend(pres.partiality) - rs_all.extend(pres.rs_set) - rh_all.extend(pres.rh_set) - sin_sq_all.extend(pres.observations.two_theta(wavelength=pres.wavelength).sin_theta_over_lambda_sq().data()) - SE_all.extend(flex.double([pres.SE]*data_size)) - wavelength_all.extend(flex.double([pres.wavelength]*data_size)) - detector_distance_set.append(pres.detector_distance_mm) - pickle_filename_all.extend(flex.std_string([pres.pickle_filename] * data_size)) - crystal_orientation_dict[pres.pickle_filename] = pres.crystal_orientation - #plot stats - self.plot_stats(filtered_results, iparams) - #write out updated crystal orientation as a pickle file - if not iparams.flag_hush: pickle.dump(crystal_orientation_dict, open(iparams.run_no+'/'+"crystal.o","wb"),pickle.HIGHEST_PROTOCOL) - #calculate average unit cell - uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(filtered_results) - unit_cell_mean = unit_cell(tuple(uc_mean)) - #recalculate stats for pr parameters - pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(filtered_results) - G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean - G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med - G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std - #from all observations merge them - crystal_symmetry = crystal.symmetry( - unit_cell=tuple(uc_mean), - space_group_symbol=str(iparams.target_space_group)) - miller_set_all=miller.set( - crystal_symmetry=crystal_symmetry, - indices=miller_indices_all, - anomalous_flag=target_anomalous_flag) - miller_array_all = miller_set_all.array( - data=I_all, - sigmas=sigI_all).set_observation_type_xray_intensity() - #sort reflections according to asymmetric-unit symmetry hkl - perm = miller_array_all.sort_permutation(by_value="packed_indices") - miller_indices_all_sort = miller_array_all.indices().select(perm) - miller_indices_ori_all_sort = miller_indices_ori_all.select(perm) - I_obs_all_sort = miller_array_all.data().select(perm) - sigI_obs_all_sort = miller_array_all.sigmas().select(perm) - G_all_sort = G_all.select(perm) - B_all_sort = B_all.select(perm) - p_all_sort = p_all.select(perm) - rs_all_sort = rs_all.select(perm) - wavelength_all_sort = wavelength_all.select(perm) - sin_sq_all_sort = sin_sq_all.select(perm) - SE_all_sort = SE_all.select(perm) - pickle_filename_all_sort = pickle_filename_all.select(perm) - miller_array_uniq = miller_array_all.merge_equivalents().array().complete_array(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max) - matches_uniq = miller.match_multi_indices( - miller_indices_unique=miller_array_uniq.indices(), - miller_indices=miller_indices_all_sort) - pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()]) - pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()]) - group_id_list = flex.int([pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))]) - tally = Counter() - for elem in group_id_list: tally[elem] += 1 - cn_group = len(tally) - #preparte txt out stat - txt_out = 'Summary of refinement and merging\n' - txt_out += ' No. good frames: %12.0f\n'%(cn_good_frame) - txt_out += ' No. bad cc frames: %12.0f\n'%(cn_bad_frame_cc) - txt_out += ' No. bad G frames) : %12.0f\n'%(cn_bad_frame_G) - txt_out += ' No. bad unit cell frames: %12.0f\n'%(cn_bad_frame_uc) - txt_out += ' No. bad gamma_e frames: %12.0f\n'%(cn_bad_frame_re) - txt_out += ' No. bad SE: %12.0f\n'%(cn_bad_frame_SE) - txt_out += ' No. observations: %12.0f\n'%(len(I_obs_all_sort)) - txt_out += 'Mean target value (BEFORE: Mean Median (Std.))\n' - txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n'%(np.mean(R_init_all), np.median(R_init_all), np.std(R_init_all)) - txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n'%(np.mean(R_xy_init_all), np.median(R_xy_init_all), np.std(R_xy_init_all)) - txt_out += 'Mean target value (AFTER: Mean Median (Std.))\n' - txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n'%(np.mean(R_final_all), np.median(R_final_all), np.std(R_final_all)) - txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n'%(np.mean(R_xy_final_all), np.median(R_xy_final_all), np.std(R_xy_final_all)) - txt_out += ' SE: %12.2f %12.2f (%9.2f)\n'%(SE_mean, SE_med, SE_std) - txt_out += ' G: %12.3e %12.3e (%9.2e)\n'%(G_mean, G_med, G_std) - txt_out += ' B: %12.2f %12.2f (%9.2f)\n'%(B_mean, B_med, B_std) - txt_out += ' Rot.x: %12.2f %12.2f (%9.2f)\n'%(rotx_mean*180/math.pi, rotx_med*180/math.pi, rotx_std*180/math.pi) - txt_out += ' Rot.y: %12.2f %12.2f (%9.2f)\n'%(roty_mean*180/math.pi, roty_med*180/math.pi, roty_std*180/math.pi) - txt_out += ' gamma_y: %12.5f %12.5f (%9.5f)\n'%(ry_mean, ry_med, ry_std) - txt_out += ' gamma_z: %12.5f %12.5f (%9.5f)\n'%(rz_mean, rz_med, rz_std) - txt_out += ' gamma_0: %12.5f %12.5f (%9.5f)\n'%(r0_mean, r0_med, r0_std) - txt_out += ' gamma_e: %12.5f %12.5f (%9.5f)\n'%(re_mean, re_med, re_std) - txt_out += ' voigt_nu: %12.5f %12.5f (%9.5f)\n'%(voigt_nu_mean, voigt_nu_med, voigt_nu_std) - txt_out += ' unit cell\n' - txt_out += ' a: %12.2f %12.2f (%9.2f)\n'%(uc_mean[0], uc_med[0], uc_std[0]) - txt_out += ' b: %12.2f %12.2f (%9.2f)\n'%(uc_mean[1], uc_med[1], uc_std[1]) - txt_out += ' c: %12.2f %12.2f (%9.2f)\n'%(uc_mean[2], uc_med[2], uc_std[2]) - txt_out += ' alpha: %12.2f %12.2f (%9.2f)\n'%(uc_mean[3], uc_med[3], uc_std[3]) - txt_out += ' beta: %12.2f %12.2f (%9.2f)\n'%(uc_mean[4], uc_med[4], uc_std[4]) - txt_out += ' gamma: %12.2f %12.2f (%9.2f)\n'%(uc_mean[5], uc_med[5], uc_std[5]) - txt_out += 'Parmeters from integration (not-refined)\n' - txt_out += ' Wavelength: %12.5f %12.5f (%9.5f)\n'%(np.mean(wavelength_all), np.median(wavelength_all), np.std(wavelength_all)) - txt_out += ' Detector distance: %12.5f %12.5f (%9.5f)\n'%(np.mean(detector_distance_set), np.median(detector_distance_set), np.std(detector_distance_set)) - txt_out += '* (standard deviation)\n' - #write out stat. pickle - if not iparams.flag_hush: - stat_dict = {"n_frames_good": [cn_good_frame], \ - "n_frames_bad_cc": [cn_bad_frame_cc], \ - "n_frames_bad_G": [cn_bad_frame_G], \ - "n_frames_bad_uc": [cn_bad_frame_uc], \ - "n_frames_bad_gamma_e": [cn_bad_frame_re], \ - "n_frames_bad_SE": [cn_bad_frame_SE], \ - "n_observations": [len(I_obs_all_sort)], \ - "R_start": [np.mean(R_init_all)], \ - "R_end": [np.mean(R_final_all)], \ - "R_xy_start": [np.mean(R_xy_init_all)], \ - "R_xy_end": [np.mean(R_xy_final_all)], \ - "mean_gamma_y": [ry_mean], \ - "std_gamma_y": [ry_std], \ - "mean_gamma_z": [rz_mean], \ - "std_gamma_z": [rz_std], \ - "mean_gamma_0": [r0_mean], \ - "std_gamma_0": [r0_std], \ - "mean_gamma_e": [re_mean], \ - "std_gamma_e": [re_std], \ - "mean_voigt_nu": [voigt_nu_mean], \ - "std_voigt_nu": [voigt_nu_std], \ - "mean_a": [uc_mean[0]], \ - "std_a": [uc_std[0]], \ - "mean_b": [uc_mean[1]], \ - "std_b": [uc_std[1]], \ - "mean_c": [uc_mean[2]], \ - "std_c": [uc_std[2]], \ - "mean_alpha": [uc_mean[3]], \ - "std_alpha": [uc_std[3]], \ - "mean_beta": [uc_mean[4]], \ - "std_beta": [uc_std[4]], \ - "mean_gamma": [uc_mean[5]], \ - "std_gamma": [uc_std[5]]} - self.write_stat_pickle(iparams, stat_dict) - return cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, \ - I_obs_all_sort, sigI_obs_all_sort,G_all_sort, B_all_sort, \ - p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, uc_mean, \ - np.mean(wavelength_all), pickle_filename_all_sort, txt_out - - def write_output(self, mdh, iparams, output_mtz_file_prefix, avg_mode): - if iparams.flag_weak_anomalous: - if avg_mode == 'final': - target_anomalous_flag = iparams.target_anomalous_flag - else: - target_anomalous_flag = False - else: - target_anomalous_flag = iparams.target_anomalous_flag - uc_mean = mdh.uc_mean - wavelength_mean = mdh.wavelength_mean - #output mtz file and report binning stat - miller_set_merge = crystal.symmetry( - unit_cell=unit_cell(tuple(uc_mean)), - space_group_symbol=str(iparams.target_space_group) - ).build_miller_set( - anomalous_flag=target_anomalous_flag, - d_min=iparams.merge.d_min) - mdh.generate_miller_array_from_miller_set(miller_set_merge, target_anomalous_flag) - miller_array_complete = miller_set_merge.array() - fake_data = flex.double([1.0]*len(miller_array_complete.indices())) - miller_array_template_asu = miller_array_complete.customized_copy(data=fake_data, \ - sigmas=fake_data).resolution_filter(d_min=iparams.merge.d_min, \ - d_max=iparams.merge.d_max) - n_refl_all = mdh.get_size() - #do another resolution filter here - i_sel_res = mdh.miller_array_merge.resolution_filter_selection(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max) - mdh.reduce_by_selection(i_sel_res) - n_refl_out_resolutions = n_refl_all - mdh.get_size() - #remove outliers - sequences = flex.int(range(mdh.get_size())) - good_sequences = [] - for i_rejection in range(iparams.n_rejection_cycle): - binner_merge = mdh.miller_array_merge.setup_binner(n_bins=200) - for i_bin in range(1, 201): - i_binner = (binner_merge.bin_indices() == i_bin) - I_obs_bin = mdh.miller_array_merge.data().select(i_binner) - sequences_bin = sequences.select(i_binner) - if len(I_obs_bin) > 0: - I_obs_bin = mdh.miller_array_merge.data().select(i_binner) - try: - i_filter = flex.abs((I_obs_bin - np.median(I_obs_bin))/np.std(I_obs_bin)) < 10 - except Exception as e: - print("Warning: outlier rejection by bins failed because of floating point.") - print(e) - i_filter = flex.bool([True]*len(I_obs_bin)) - good_sequences.extend(list(sequences_bin.select(i_filter))) - mdh.reduce_by_selection(flex.size_t(good_sequences)) - n_refl_outliers = n_refl_all - n_refl_out_resolutions - mdh.get_size() - #get iso if given. - mxh = mx_handler() - flag_hklisoin_found, miller_array_iso = mxh.get_miller_array_from_reflection_file(iparams.hklisoin) - #write output files - if output_mtz_file_prefix != '': - #write as mtz file - miller_array_merge_unique = mdh.miller_array_merge.merge_equivalents().array() - info = miller.array_info(wavelength=wavelength_mean) - miller_array_merge_unique.set_info(info) - mtz_dataset_merge = miller_array_merge_unique.as_mtz_dataset(column_root_label="IOBS") - mtz_dataset_merge.mtz_object().write(file_name=output_mtz_file_prefix+'_merge.mtz') - #write as cns file - f_cns = open(output_mtz_file_prefix+'_merge.hkl', 'w') - miller_array_merge_unique.export_as_cns_hkl(file_object=f_cns) - f_cns.close() - #calculate merging stat table - if True: - #calculate isotropic B-factor - try: - mxh = mx_handler() - asu_contents = mxh.get_asu_contents(iparams.n_residues) - observations_as_f = mdh.miller_array_merge.as_amplitude_array() - observations_as_f.setup_binner(auto_binning=True) - wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) - B_merged = wp.wilson_b - except Exception as e: - B_merged = 0 - print("Warning: b-factor calculation in mod_util failed. Reset b-factor to 0") - print(e) - #report binning stats - txt_out = '\n' - txt_out += 'Isotropic B-factor: %7.2f\n'%(B_merged) - txt_out += 'No. of reflections\n' - txt_out += ' all: %7.0f\n'%(n_refl_all) - txt_out += ' outside resolution: %7.0f\n'%(n_refl_out_resolutions) - txt_out += ' outliers: %7.0f\n'%(n_refl_outliers) - txt_out += ' total left: %7.0f\n'%(mdh.get_size()) - txt_out += 'Summary for '+output_mtz_file_prefix+'_merge.mtz\n' - txt_out += 'Bin Resolution Range Completeness |Rmerge Rsplit CC1/2 N_ind |CCiso N_ind|CCanoma N_ind| \n' - txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n' - #for stat pickle - sp_res,sp_complete,sp_n_obs,sp_cc12,sp_cc12_anom,sp_rmerge,sp_i_o_sigi,sp_isqr = ([],[],[],[],[],[],[],[]) - #binning - binner_template_asu = miller_array_template_asu.setup_binner(n_bins=iparams.n_bins) - binner_template_asu_indices = binner_template_asu.bin_indices() - #for stats on axis cones - mdh_astar = deepcopy(mdh) - mdh_bstar = deepcopy(mdh) - mdh_cstar = deepcopy(mdh) - mdh_astar.reduce_to_cone_on_axis((1,0,0), iparams.percent_cone_fraction) - mdh_bstar.reduce_to_cone_on_axis((0,1,0), iparams.percent_cone_fraction) - mdh_cstar.reduce_to_cone_on_axis((0,0,1), iparams.percent_cone_fraction) - #prepare text out for axis cones - txt_out_cone = 'Summary of CC1/2 on three crystal axes\n' - txt_out_cone += 'Bin Resolution Range CC1/2 N_refl \n' - txt_out_cone += ' a* b* c* | a* b* c* | a* b* c* \n' - txt_out_cone += '---------------------------------------------------------------------------------------------------------\n' - for i in range(1,iparams.n_bins+1): - i_binner = (binner_template_asu_indices == i) - miller_indices_template_bin = miller_array_template_asu.indices().select(i_binner) - #for all reflections - mdh_bin = deepcopy(mdh) - mdh_bin.reduce_by_miller_index(miller_indices_template_bin) - cc12, n_refl_cc12 = mdh_bin.get_cc12() - cciso, n_refl_cciso = mdh_bin.get_cciso(miller_array_iso) - cc_anom_acentric, n_refl_anom_acentric = mdh_bin.get_cc_anom() - completeness = (mdh_bin.get_size()/len(miller_indices_template_bin))*100 - multiplicity = mdh_bin.get_multiplicity() - txt_out += '%02d %7.2f - %7.2f %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \ - %(i, binner_template_asu.bin_d_range(i)[0], binner_template_asu.bin_d_range(i)[1], \ - completeness, \ - mdh_bin.get_size(), len(miller_indices_template_bin),\ - multiplicity, mdh_bin.get_r_meas()*100, mdh_bin.get_r_split()*100, \ - cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \ - cc_anom_acentric, n_refl_anom_acentric, \ - mdh_bin.get_mean_IoversigI(), mdh_bin.get_mean_I(), mdh_bin.get_mean_sigI(), mdh_bin.get_second_moment()) - #for reflections on cones - mdh_astar_bin = deepcopy(mdh_astar) - mdh_astar_bin.reduce_by_miller_index(miller_indices_template_bin) - cc12_astar, n_refl_cc12_astar = mdh_astar_bin.get_cc12() - mdh_bstar_bin = deepcopy(mdh_bstar) - mdh_bstar_bin.reduce_by_miller_index(miller_indices_template_bin) - cc12_bstar, n_refl_cc12_bstar = mdh_bstar_bin.get_cc12() - mdh_cstar_bin = deepcopy(mdh_cstar) - mdh_cstar_bin.reduce_by_miller_index(miller_indices_template_bin) - cc12_cstar, n_refl_cc12_cstar = mdh_cstar_bin.get_cc12() - txt_out_cone += '%02d %7.2f - %7.2f %7.2f %7.2f %7.2f %10.1f %10.1f %10.1f %6.0f %6.0f %6.0f\n' \ - %(i, binner_template_asu.bin_d_range(i)[0], binner_template_asu.bin_d_range(i)[1], \ - cc12_astar*100, cc12_bstar*100, cc12_cstar*100, \ - mdh_astar_bin.get_mean_I(), mdh_bstar_bin.get_mean_I(), mdh_cstar_bin.get_mean_I(), \ - n_refl_cc12_astar, n_refl_cc12_bstar, n_refl_cc12_cstar) - #for stat pickle - sp_res.append(binner_template_asu.bin_d_range(i)[1]) - sp_complete.append(completeness) - sp_n_obs.append(multiplicity) - sp_cc12.append(cc12) - sp_cc12_anom.append(cc_anom_acentric) - sp_rmerge.append(mdh_bin.get_r_meas()*100) - sp_i_o_sigi.append(mdh_bin.get_mean_IoversigI()) - sp_isqr.append(mdh.get_second_moment()) - #txt out total for all reflections - cc12, n_refl_cc12 = mdh.get_cc12() - cciso, n_refl_cciso = mdh.get_cciso(miller_array_iso) - cc_anom_acentric, n_refl_anom_acentric = mdh.get_cc_anom() - txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n' - txt_out += ' TOTAL %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \ - %((mdh.get_size()/miller_array_template_asu.size())*100, \ - mdh.get_size(), miller_array_template_asu.size(),\ - mdh.get_multiplicity(), mdh.get_r_meas()*100, mdh.get_r_split()*100, \ - cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \ - cc_anom_acentric, n_refl_anom_acentric, \ - mdh.get_mean_IoversigI(), mdh.get_mean_I(), mdh.get_mean_sigI(), mdh.get_second_moment()) - txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n' - txt_out += '\n' - #txt out total for reflections on cones - cc12_astar, n_refl_cc12_astar = mdh_astar.get_cc12() - cc12_bstar, n_refl_cc12_bstar = mdh_bstar.get_cc12() - cc12_cstar, n_refl_cc12_cstar = mdh_cstar.get_cc12() - txt_out_cone += '----------------------------------------------------------------------------------------------------------\n' - txt_out_cone += ' total %7.2f %7.2f %7.2f %10.1f %10.1f %10.1f %6.0f %6.0f %6.0f\n' \ - %(cc12_astar*100, cc12_bstar*100, cc12_cstar*100, \ - mdh_astar.get_mean_I(), mdh_bstar.get_mean_I(), mdh_cstar.get_mean_I(), \ - n_refl_cc12_astar, n_refl_cc12_bstar, n_refl_cc12_cstar) - txt_out_cone += '----------------------------------------------------------------------------------------------------------\n' - txt_out_cone += '\n' - txt_out_table1 = "Table1 ("+avg_mode+")\n" - txt_out_table1 += " Space group: "+str(mdh.miller_array_merge.space_group_info())+"\n" - txt_out_table1 += " Cell dimensions: %6.2f, %6.2f, %6.2f, %6.2f, %6.2f, %6.2f\n"%tuple(mdh.uc_mean) - txt_out_table1 += " Resolution (A): %6.2f - %6.2f (%6.2f - %6.2f)\n"%(mdh.miller_array_merge.d_max_min()[0], mdh.miller_array_merge.d_max_min()[1], sp_res[-2], sp_res[-1]) - txt_out_table1 += " Rmerge: %6.2f (%6.2f)\n"%(mdh.get_r_meas()*100, sp_rmerge[-1]) - txt_out_table1 += " CC1/2: %6.2f (%6.2f)\n"%(mdh.get_cc12()[0]*100, sp_cc12[-1]) - txt_out_table1 += " I/sigI: %6.2f (%6.2f)\n"%(mdh.get_mean_IoversigI(), sp_i_o_sigi[-1]) - txt_out_table1 += " Completeness (%%): %6.2f (%6.2f)\n"%((mdh.get_size()/miller_array_template_asu.size())*100, sp_complete[-1]) - txt_out_table1 += " Redundancy: %6.2f (%6.2f)\n"%(mdh.get_multiplicity(), sp_n_obs[-1]) - #save data for stat. pickle in stat_dict - if not iparams.flag_hush: - stat_dict = {"binned_resolution": [sp_res], \ - "binned_completeness": [sp_complete], \ - "binned_n_obs": [sp_n_obs], \ - "binned_cc12": [sp_cc12], \ - "binned_cc12_anom": [sp_cc12_anom], \ - "binned_rmerge": [sp_rmerge], \ - "binned_i_o_sigi": [sp_i_o_sigi], \ - "binned_isqr": [sp_isqr], \ - "total_res_max": [mdh.miller_array_merge.d_max_min()[0]], \ - "total_res_min": [mdh.miller_array_merge.d_max_min()[1]], \ - "total_completeness": [(mdh.get_size()/miller_array_template_asu.size())*100], \ - "total_n_obs": [mdh.get_multiplicity()], \ - "total_cc12": [mdh.get_cc12()[0]*100], \ - "total_rmerge": [mdh.get_r_meas()*100], \ - "total_i_o_sigi": [mdh.get_mean_IoversigI()], \ - "space_group_info": [mdh.miller_array_merge.space_group_info()], \ - } - self.write_stat_pickle(iparams, stat_dict) - txt_out += txt_out_cone + txt_out_table1 - return mdh, txt_out - - def plot_stats(self, results, iparams): - #retrieve stats from results and plot them - if iparams.flag_plot or iparams.flag_output_verbose: - #for plotting set n_bins = 5 to avoid empty bin - n_bins_plot = 5 - #get expected f^2 - try: - mxh = mx_handler() - asu_contents = mxh.get_asu_contents(iparams.n_residues) - observations_as_f = results[0].observations.as_amplitude_array() - binner_template_asu = observations_as_f.setup_binner(n_bins=n_bins_plot) - wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) - expected_f_sq = wp.expected_f_sq - mean_stol_sq = wp.mean_stol_sq - except Exception: - expected_f_sq = flex.double([0]*n_bins_plot) - mean_stol_sq = flex.double(range(n_bins_plot)) - print("Warning: Wilson plot calculation in plot stats failed.") - #setup list - params_array = np.array([[pres.R_init, pres.R_final, pres.R_xy_init, pres.R_xy_final, \ - pres.G, pres.B, pres.rotx*180/math.pi, pres.roty*180/math.pi, \ - pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, \ - pres.uc_params[0], pres.uc_params[1], pres.uc_params[2], \ - pres.uc_params[3], pres.uc_params[4], pres.uc_params[5], \ - pres.CC_final, pres.pickle_filename] for pres in results]) - params = ['Rinit','Rfinal','Rxyinit', 'Rxyfinal', \ - 'G','B','rot_x','rot_y','gamma_y','gamma_z','gamma_0','gamma_e','voigtnu' , \ - 'a','b','c','alpha','beta','gamma','CC','Filename'] - #keep parameter history if verbose is selected - if iparams.flag_output_verbose: - fileseq_list = flex.int() - for file_in in os.listdir(iparams.run_no): - if file_in.endswith('.paramhist'): - file_split = file_in.split('.') - fileseq_list.append(int(file_split[0])) - if len(fileseq_list) == 0: - new_fileseq = 0 - else: - new_fileseq = flex.max(fileseq_list) + 1 - newfile_name = str(new_fileseq) + '.paramhist' - txt_out_verbose = '\n'.join([' '.join(p) for p in params_array]) - f = open(iparams.run_no+'/'+newfile_name, 'w') - f.write(txt_out_verbose) - f.close() - #plotting - if iparams.flag_plot: - try: - import matplotlib.pyplot as plt - except Exception as e: - print("Warning: error importing matplotlib.pyplot") - print(e) - return - n_rows = 3 - n_cols = int(math.ceil(len(params)/n_rows)) - num_bins = 10 - for i in range(len(params)-1): - tmp_params = params_array[:,i].astype(np.float) - plt.subplot(n_rows,n_cols,i+1) - plt.hist(tmp_params, num_bins, normed=0, facecolor='green', alpha=0.5) - plt.ylabel('Frequencies') - plt.title(params[i]+'\nmu %5.1f med %5.1f sigma %5.1f' %(np.mean(tmp_params), np.median(tmp_params), np.std(tmp_params))) - plt.show() - - def combine_pre_merge(self, result, iparams): - mi_all = flex.miller_index() - mio_all = flex.miller_index() - I_all = flex.double() - sigI_all = flex.double() - G_all = flex.double() - B_all = flex.double() - p_all = flex.double() - rs_all = flex.double() - wavelength_all = flex.double() - sin_all = flex.double() - SE_all = flex.double() - uc_mean_set = [] - wavelength_mean_set = [] - pickle_filename_all = flex.std_string() - for res in result: - for prep_output in res: - _, _, mi, mio, I, sigI, G, B, p, rs, wavelength, sin, SE, uc_mean, wavelength_mean, pickle_filename_set, txt_out = prep_output - mi_all.extend(mi) - mio_all.extend(mio) - I_all.extend(I) - sigI_all.extend(sigI) - G_all.extend(G) - B_all.extend(B) - p_all.extend(p) - rs_all.extend(rs) - wavelength_all.extend(wavelength) - sin_all.extend(sin) - SE_all.extend(SE) - uc_mean_set.extend(uc_mean) - wavelength_mean_set.append(wavelength_mean) - pickle_filename_all.extend(pickle_filename_set) - uc_mean = np.mean(np.array(uc_mean_set).reshape(-1,6), axis=0) - wavelength_mean = np.mean(wavelength_mean_set) - ms_template = crystal.symmetry( - unit_cell=tuple(uc_mean), - space_group_symbol=str(iparams.target_space_group) - ).build_miller_set( - anomalous_flag=iparams.target_anomalous_flag, - d_min=iparams.merge.d_min) - ma_all = ms_template.array().customized_copy(indices=mi_all, data=I_all, sigmas=sigI_all) - #sort reflections according to asymmetric-unit symmetry hkl - perm = ma_all.sort_permutation(by_value="packed_indices") - mi_all_sort = mi_all.select(perm) - mio_all_sort = mio_all.select(perm) - I_all_sort = I_all.select(perm) - sigI_all_sort = sigI_all.select(perm) - G_all_sort = G_all.select(perm) - B_all_sort = B_all.select(perm) - p_all_sort = p_all.select(perm) - rs_all_sort = rs_all.select(perm) - wavelength_all_sort = wavelength_all.select(perm) - sin_all_sort = sin_all.select(perm) - SE_all_sort = SE_all.select(perm) - pickle_filename_all_sort = pickle_filename_all.select(perm) - ma_uniq = ma_all.merge_equivalents().array().complete_array(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max) - matches_uniq = miller.match_multi_indices( - miller_indices_unique=ma_uniq.indices(), - miller_indices=mi_all_sort) - pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()]) - pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()]) - group_id_list = flex.int([pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))]) - tally = Counter() - for elem in group_id_list: tally[elem] += 1 - cn_group = len(tally) - return cn_group, group_id_list, mi_all_sort, mio_all_sort, \ - I_all_sort, sigI_all_sort, G_all_sort, B_all_sort, \ - p_all_sort, rs_all_sort, wavelength_all_sort, sin_all_sort, SE_all_sort, uc_mean, \ - wavelength_mean, pickle_filename_all_sort, "" diff --git a/prime/postrefine/postrefine.py b/prime/postrefine/postrefine.py deleted file mode 100644 index dea0e1de5d1..00000000000 --- a/prime/postrefine/postrefine.py +++ /dev/null @@ -1,444 +0,0 @@ -from __future__ import absolute_import, division, print_function -from cctbx.array_family import flex -from cctbx import miller, crystal -from six.moves import cPickle as pickle -from .mod_leastsqr import leastsqr_handler -from .mod_results import postref_results -from cctbx.crystal import symmetry -from scitbx.matrix import sqr -from cctbx import statistics -from .mod_partiality import partiality_handler -from .mod_lbfgs_partiality import lbfgs_partiality_handler -from .mod_mx import mx_handler -import math, os -from .mod_input import read_frame -from six.moves import range -from six.moves import zip - -class postref_handler(object): - """ - handle post-refinement - - read-in and store input in input_handler object - - generate a mean-intensity-scaled mtz file as a reference set - - perform post-refinement - """ - def __init__(self): - """ - Constructor - """ - - def organize_input(self, observations_pickle, iparams, avg_mode, pickle_filename=None): - """Given the pickle file, extract and prepare observations object and - the alpha angle (meridional to equatorial). - """ - #get general parameters - if iparams.isoform_name is not None: - if "identified_isoform" not in observations_pickle: - return None, "No identified isoform" - if observations_pickle["identified_isoform"] != iparams.isoform_name: - return None, "Identified isoform(%s) is not the requested isoform (%s)"%(observations_pickle["identified_isoform"], iparams.isoform_name) - if iparams.flag_weak_anomalous: - if avg_mode == 'final': - target_anomalous_flag = iparams.target_anomalous_flag - else: - target_anomalous_flag = False - else: - target_anomalous_flag = iparams.target_anomalous_flag - img_filename_only = '' - if pickle_filename: img_filename_only = os.path.basename(pickle_filename) - txt_exception = ' {0:40} ==> '.format(img_filename_only) - #for dials integration pickles - also look for experimentxxx.json - if "miller_index" in observations_pickle: - from dxtbx.model.experiment_list import ExperimentListFactory - exp_json_file = os.path.join(os.path.dirname(pickle_filename),img_filename_only.split('_')[0]+'_refined_experiments.json') - if os.path.isfile(exp_json_file): - experiments = ExperimentListFactory.from_json_file(exp_json_file) - dials_crystal = experiments[0].crystal - detector = experiments[0].detector - beam = experiments[0].beam - crystal_symmetry = crystal.symmetry( - unit_cell=dials_crystal.get_unit_cell().parameters(), - space_group_symbol=str(iparams.target_space_group)) - miller_set_all=miller.set( - crystal_symmetry=crystal_symmetry, - indices=observations_pickle['miller_index'], - anomalous_flag=target_anomalous_flag) - observations = miller_set_all.array( - data=observations_pickle['intensity.sum.value'], - sigmas=flex.sqrt(observations_pickle['intensity.sum.variance'])).set_observation_type_xray_intensity() - detector_distance_mm = detector[0].get_distance() - alpha_angle_obs = flex.double([0]*len(observations.data())) - wavelength = beam.get_wavelength() - spot_pred_x_mm = observations_pickle['s1'] #a disguise of s1 - spot_pred_y_mm = flex.double([0]*len(observations.data())) - #calculate the crystal orientation - O = sqr(dials_crystal.get_unit_cell().orthogonalization_matrix()).transpose() - R = sqr(dials_crystal.get_U()).transpose() - from cctbx.crystal_orientation import crystal_orientation, basis_type - crystal_init_orientation = crystal_orientation(O*R, basis_type.direct) - else: - txt_exception += exp_json_file+' not found' - print(txt_exception) - return None, txt_exception - else: - #for cctbx.xfel proceed as usual - observations = observations_pickle["observations"][0] - detector_distance_mm = observations_pickle['distance'] - mm_predictions = iparams.pixel_size_mm*(observations_pickle['mapped_predictions'][0]) - xbeam = observations_pickle["xbeam"] - ybeam = observations_pickle["ybeam"] - alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ - for pred in mm_predictions]) - spot_pred_x_mm = flex.double([pred[0]-xbeam for pred in mm_predictions]) - spot_pred_y_mm = flex.double([pred[1]-ybeam for pred in mm_predictions]) - #Polarization correction - wavelength = observations_pickle["wavelength"] - crystal_init_orientation = observations_pickle["current_orientation"][0] - #continue reading... - if iparams.flag_LP_correction and "observations" in observations_pickle: - fx = 1 - iparams.polarization_horizontal_fraction - fy = 1 - fx - if fx > 1.0 or fx < 0: - print('Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1') - print('No polarization correction. Continue with post-refinement') - else: - phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \ - for pred in mm_predictions]) - bragg_angle_obs = observations.two_theta(wavelength).data() - P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\ - (fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))) - I_prime = observations.data()/P - sigI_prime =observations.sigmas()/P - observations = observations.customized_copy(data=flex.double(I_prime), - sigmas=flex.double(sigI_prime)) - #set observations with target space group - !!! required for correct - #merging due to map_to_asu command. - if iparams.target_crystal_system is not None: - target_crystal_system = iparams.target_crystal_system - else: - target_crystal_system = observations.crystal_symmetry().space_group().crystal_system() - lph = lbfgs_partiality_handler() - if iparams.flag_override_unit_cell: - uc_constrained_inp = lph.prep_input(iparams.target_unit_cell.parameters(), target_crystal_system) - else: - uc_constrained_inp = lph.prep_input(observations.unit_cell().parameters(), target_crystal_system) - uc_constrained = list(lph.prep_output(uc_constrained_inp, target_crystal_system)) - try: - #apply constrain using the crystal system - miller_set = symmetry( - unit_cell=uc_constrained, - space_group_symbol=str(iparams.target_space_group) - ).build_miller_set( - anomalous_flag=target_anomalous_flag, - d_min=iparams.merge.d_min) - observations = observations.customized_copy(anomalous_flag=target_anomalous_flag, - crystal_symmetry=miller_set.crystal_symmetry()) - except Exception: - a,b,c,alpha,beta,gamma = uc_constrained - txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)'%(a,b,c,alpha,beta,gamma) - print(txt_exception) - return None, txt_exception - #reset systematic absence - sys_absent_negate_flags = flex.bool([sys_absent_flag[1]==False for sys_absent_flag in observations.sys_absent_flags()]) - observations = observations.select(sys_absent_negate_flags) - alpha_angle_obs = alpha_angle_obs.select(sys_absent_negate_flags) - spot_pred_x_mm = spot_pred_x_mm.select(sys_absent_negate_flags) - spot_pred_y_mm = spot_pred_y_mm.select(sys_absent_negate_flags) - - #remove observations from rejection list - if iparams.rejections: - if pickle_filename in iparams.rejections: - miller_indices_ori_rejected = iparams.rejections[pickle_filename] - i_sel_flag = flex.bool([True]*len(observations.data())) - cnrej = 0 - for miller_index_ori_rejected in miller_indices_ori_rejected: - for i_index_ori, miller_index_ori in enumerate(observations.indices()): - if miller_index_ori_rejected == miller_index_ori: - i_sel_flag[i_index_ori] = False - cnrej += 1 - observations = observations.customized_copy(indices=observations.indices().select(i_sel_flag), - data=observations.data().select(i_sel_flag), - sigmas=observations.sigmas().select(i_sel_flag)) - alpha_angle_obs = alpha_angle_obs.select(i_sel_flag) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel_flag) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel_flag) - - #filter resolution - i_sel_res = observations.resolution_filter_selection(d_max=iparams.merge.d_max, d_min=iparams.merge.d_min) - observations = observations.select(i_sel_res) - alpha_angle_obs = alpha_angle_obs.select(i_sel_res) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res) - - #Filter weak - i_sel = (observations.data()/observations.sigmas()) > iparams.merge.sigma_min - observations = observations.select(i_sel) - alpha_angle_obs = alpha_angle_obs.select(i_sel) - spot_pred_x_mm = spot_pred_x_mm.select(i_sel) - spot_pred_y_mm = spot_pred_y_mm.select(i_sel) - - #filter icering (if on) - if iparams.icering.flag_on: - miller_indices = flex.miller_index() - I_set = flex.double() - sigI_set = flex.double() - alpha_angle_obs_set = flex.double() - spot_pred_x_mm_set = flex.double() - spot_pred_y_mm_set = flex.double() - for miller_index, d, I, sigI, alpha, spot_x, spot_y in zip(observations.indices(), observations.d_spacings().data(), - observations.data(), observations.sigmas(), alpha_angle_obs, - spot_pred_x_mm, spot_pred_y_mm): - if d > iparams.icering.d_upper or d < iparams.icering.d_lower: - miller_indices.append(miller_index) - I_set.append(I) - sigI_set.append(sigI) - alpha_angle_obs_set.append(alpha) - spot_pred_x_mm_set.append(spot_x) - spot_pred_y_mm_set.append(spot_y) - observations = observations.customized_copy(indices=miller_indices, - data=I_set, sigmas=sigI_set) - alpha_angle_obs = alpha_angle_obs_set[:] - spot_pred_x_mm = spot_pred_x_mm_set[:] - spot_pred_y_mm = spot_pred_y_mm_set[:] - #replacing sigI (if set) - if iparams.flag_replace_sigI: - observations = observations.customized_copy(sigmas=flex.sqrt(observations.data())) - inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation - return inputs, 'OK' - - def get_observations_non_polar(self, observations_original, pickle_filename, iparams): - #return observations with correct polarity - if iparams.indexing_ambiguity.index_basis_in is None: - return observations_original.map_to_asu(), 'h,k,l' - ind_pickle = iparams.indexing_ambiguity.index_basis_in - if pickle_filename not in ind_pickle: - return observations_original.map_to_asu(), 'Not Found' - from cctbx import sgtbx - cb_op = sgtbx.change_of_basis_op(ind_pickle[pickle_filename]) - observations_alt = observations_original.map_to_asu().change_basis(cb_op).map_to_asu() - return observations_alt, ind_pickle[pickle_filename] - - def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode): - #1. Prepare data - observations_pickle = read_frame(pickle_filename) - pickle_filepaths = pickle_filename.split('/') - img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] - txt_exception = ' {0:40} ==> '.format(img_filename_only) - if observations_pickle is None: - txt_exception += 'empty or bad input file\n' - return None, txt_exception - inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) - if inputs is not None: - observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs - else: - txt_exception += txt_organize_input + '\n' - return None, txt_exception - #2. Select data for post-refinement (only select indices that are common with the reference set - observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) - matches = miller.match_multi_indices( - miller_indices_unique=miller_array_ref.indices(), - miller_indices=observations_non_polar.indices()) - pair_0 = flex.size_t([pair[0] for pair in matches.pairs()]) - pair_1 = flex.size_t([pair[1] for pair in matches.pairs()]) - references_sel = miller_array_ref.select(pair_0) - observations_original_sel = observations_original.select(pair_1) - observations_non_polar_sel = observations_non_polar.select(pair_1) - alpha_angle_set = alpha_angle.select(pair_1) - spot_pred_x_mm_set = spot_pred_x_mm.select(pair_1) - spot_pred_y_mm_set = spot_pred_y_mm.select(pair_1) - #4. Do least-squares refinement - lsqrh = leastsqr_handler() - try: - refined_params, stats, n_refl_postrefined = lsqrh.optimize(references_sel.data(), - observations_original_sel, wavelength, - crystal_init_orientation, alpha_angle_set, - spot_pred_x_mm_set, spot_pred_y_mm_set, - iparams, - pres_in, - observations_non_polar_sel, - detector_distance_mm) - except Exception: - txt_exception += 'optimization failed.\n' - return None, txt_exception - #caculate partiality for output (with target_anomalous check) - G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \ - a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params - inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) - observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs - observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) - from cctbx.uctbx import unit_cell - uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) - if pres_in is not None: - crystal_init_orientation = pres_in.crystal_orientation - two_theta = observations_original.two_theta(wavelength=wavelength).data() - ph = partiality_handler() - partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(uc_fin, rotx_fin, roty_fin, - observations_original.indices(), - ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, - two_theta, alpha_angle, wavelength, - crystal_init_orientation, - spot_pred_x_mm, spot_pred_y_mm, - detector_distance_mm, - iparams.partiality_model, - iparams.flag_beam_divergence) - #calculate the new crystal orientation - O = sqr(uc_fin.orthogonalization_matrix()).transpose() - R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() - from cctbx.crystal_orientation import crystal_orientation, basis_type - CO = crystal_orientation(O*R, basis_type.direct) - crystal_fin_orientation = CO.rotate_thru((1,0,0), rotx_fin - ).rotate_thru((0,1,0), roty_fin) - #remove reflections with partiality below threshold - i_sel = partiality_fin > iparams.merge.partiality_min - partiality_fin_sel = partiality_fin.select(i_sel) - rs_fin_sel = rs_fin.select(i_sel) - rh_fin_sel = rh_fin.select(i_sel) - observations_non_polar_sel = observations_non_polar.customized_copy(\ - indices=observations_non_polar.indices().select(i_sel), - data=observations_non_polar.data().select(i_sel), - sigmas=observations_non_polar.sigmas().select(i_sel)) - observations_original_sel = observations_original.customized_copy(\ - indices=observations_original.indices().select(i_sel), - data=observations_original.data().select(i_sel), - sigmas=observations_original.sigmas().select(i_sel)) - pres = postref_results() - pres.set_params(observations = observations_non_polar_sel, - observations_original = observations_original_sel, - refined_params=refined_params, - stats=stats, - partiality=partiality_fin_sel, - rs_set=rs_fin_sel, - rh_set=rh_fin_sel, - frame_no=frame_no, - pickle_filename=pickle_filename, - wavelength=wavelength, - crystal_orientation=crystal_fin_orientation, - detector_distance_mm=detector_distance_mm) - r_change = ((pres.R_final - pres.R_init)/pres.R_init)*100 - r_xy_change = ((pres.R_xy_final - pres.R_xy_init)/pres.R_xy_init)*100 - cc_change = ((pres.CC_final - pres.CC_init)/pres.CC_init)*100 - txt_postref= '{0:40} => RES:{1:5.2f} NREFL:{2:5d} R:{3:6.1f}% RXY:{4:5.1f}% CC:{5:5.1f}% G:{6:6.4f} B:{7:5.1f} CELL:{8:6.1f}{9:6.1f} {10:6.1f} {11:5.1f} {12:5.1f} {13:5.1f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin) - print(txt_postref) - txt_postref += '\n' - return pres, txt_postref - - def calc_mean_intensity(self, pickle_filename, iparams, avg_mode): - observations_pickle = read_frame(pickle_filename) - pickle_filepaths = pickle_filename.split('/') - txt_exception = ' {0:40} ==> '.format(pickle_filepaths[len(pickle_filepaths)-1]) - if observations_pickle is None: - txt_exception += 'empty or bad input file\n' - return None, txt_exception - inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) - if inputs is not None: - observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs - else: - txt_exception += txt_organize_input + '\n' - return None, txt_exception - #filter resolution - observations_sel = observations_original.resolution_filter(d_min=iparams.scale.d_min, d_max=iparams.scale.d_max) - #filer sigma - i_sel = (observations_sel.data()/observations_sel.sigmas()) > iparams.scale.sigma_min - if len(observations_sel.data().select(i_sel)) == 0: - return None, txt_exception - mean_I = flex.median(observations_sel.data().select(i_sel)) - return mean_I, txt_exception+'ok' - - def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams, mean_of_mean_I, avg_mode): - observations_pickle = read_frame(pickle_filename) - pickle_filepaths = pickle_filename.split('/') - img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] - txt_exception = ' {0:40} ==> '.format(img_filename_only) - if observations_pickle is None: - txt_exception += 'empty or bad input file\n' - return None, txt_exception - inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) - if inputs is not None: - observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs - else: - txt_exception += txt_organize_input + '\n' - return None, txt_exception - #select only reflections matched with scale input params. - #filter by resolution - i_sel_res = observations_original.resolution_filter_selection(d_min=iparams.scale.d_min, - d_max=iparams.scale.d_max) - observations_original_sel = observations_original.select(i_sel_res) - alpha_angle_sel = alpha_angle.select(i_sel_res) - spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res) - spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res) - #filter by sigma - i_sel_sigmas = (observations_original_sel.data()/observations_original_sel.sigmas()) > iparams.scale.sigma_min - observations_original_sel = observations_original_sel.select(i_sel_sigmas) - alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas) - spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas) - spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas) - observations_non_polar_sel, index_basis_name = self.get_observations_non_polar(observations_original_sel, pickle_filename, iparams) - observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) - uc_params = observations_original.unit_cell().parameters() - ph = partiality_handler() - r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), - observations_original_sel.indices(), wavelength) - #calculate first G - (G, B) = (1,0) - stats = (0,0,0,0,0,0,0,0,0,0) - if mean_of_mean_I > 0: - G = flex.median(observations_original_sel.data())/mean_of_mean_I - if iparams.flag_apply_b_by_frame: - try: - mxh = mx_handler() - asu_contents = mxh.get_asu_contents(iparams.n_residues) - observations_as_f = observations_non_polar_sel.as_amplitude_array() - binner_template_asu = observations_as_f.setup_binner(auto_binning=True) - wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) - G = wp.wilson_intensity_scale_factor * 1e2 - B = wp.wilson_b - except Exception: - txt_exception += 'warning B-factor calculation failed.\n' - return None, txt_exception - two_theta = observations_original.two_theta(wavelength=wavelength).data() - sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() - ry, rz, re, voigt_nu, rotx, roty = (0, 0, iparams.gamma_e, iparams.voigt_nu, 0, 0) - partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ - crystal_init_orientation.unit_cell(), - rotx, roty, observations_original.indices(), - ry, rz, r0, re, voigt_nu, - two_theta, alpha_angle, wavelength, - crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, - detector_distance_mm, iparams.partiality_model, - iparams.flag_beam_divergence) - if iparams.flag_plot_expert: - n_bins = 20 - binner = observations_original.setup_binner(n_bins=n_bins) - binner_indices = binner.bin_indices() - avg_partiality_init = flex.double() - avg_rs_init = flex.double() - avg_rh_init = flex.double() - one_dsqr_bin = flex.double() - for i in range(1,n_bins+1): - i_binner = (binner_indices == i) - if len(observations_original.data().select(i_binner)) > 0: - print(binner.bin_d_range(i)[1], flex.mean(partiality_init.select(i_binner)), flex.mean(rs_init.select(i_binner)), flex.mean(rh_init.select(i_binner)), len(partiality_init.select(i_binner))) - #monte-carlo merge - if iparams.flag_monte_carlo: - G = 1 - B = 0 - partiality_init=flex.double([1]*len(partiality_init)) - #save results - refined_params = flex.double([G,B,rotx,roty,ry,rz,r0,re,voigt_nu,uc_params[0],uc_params[1],uc_params[2],uc_params[3],uc_params[4],uc_params[5]]) - pres = postref_results() - pres.set_params(observations = observations_non_polar, - observations_original = observations_original, - refined_params=refined_params, - stats=stats, - partiality=partiality_init, - rs_set=rs_init, - rh_set=rh_init, - frame_no=frame_no, - pickle_filename=pickle_filename, - wavelength=wavelength, - crystal_orientation=crystal_init_orientation, - detector_distance_mm=detector_distance_mm) - txt_scale_frame_by_mean_I = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:6.4f} B:{4:6.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original.d_min(), len(observations_original_sel.data()), G, B, uc_params[0],uc_params[1],uc_params[2],uc_params[3],uc_params[4],uc_params[5]) - print(txt_scale_frame_by_mean_I) - txt_scale_frame_by_mean_I += '\n' - return pres, txt_scale_frame_by_mean_I diff --git a/prime/postrefine/whiteboard.tex b/prime/postrefine/whiteboard.tex deleted file mode 100644 index f28acc34417..00000000000 --- a/prime/postrefine/whiteboard.tex +++ /dev/null @@ -1,155 +0,0 @@ -\documentclass[12pt, letterpaper]{article} -\usepackage[english]{babel} -\usepackage{amsmath} -\usepackage{amssymb} -\usepackage{accents} - - %%%%%%%%%%%% - % PREAMBLE % - %%%%%%%%%%%% -\begin{document} -\selectlanguage{english} - -\section{Whiteboard 2013-09-10} - -\begin{equation} - f = \frac{1}{10} \sum_{i} \limits - ( G p I_{\mathrm{calc}} + K - I_{\mathrm{obs}})^{2} -\end{equation} -where $g_{i} = G p I_{\mathrm{calc}} + K - I_{\mathrm{obs}}$ is -\texttt{func\_callable} - -\begin{equation} - \frac{\partial g}{\partial \theta_{x}} - = ( G I_{\mathrm{c}} ) \frac{\partial p}{\partial \theta_{x}} -\end{equation} -where $\partial g / \partial \theta_{x}$ is \texttt{jacobian\_callable} - -\begin{equation} - \frac{\partial f}{\partial \theta_{x}} - = 2 \sum (G p I_{c} + K - I_{o}) (G I_{c}) - \frac{\partial p}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - \frac{\partial f}{\partial K} - = 2 \sum_{i} \limits g_{i} \frac{\partial g_{i}}{\partial K} - = 2 \sum_{i} \limits g_{i} -\end{equation} - -\begin{equation} - \frac{\partial f}{\partial G} - = 2 \sum_{i} \limits g_{i} \frac{\partial g_{i}}{\partial G} - = 2 \sum_{i} \limits g_{i} p I_{\mathrm{calc}} -\end{equation} - -\begin{equation} - \frac{\partial g_{i}}{\partial \theta_{x}}, - \frac{\partial g_{i}}{\partial \xi}, - \ldots -\end{equation} - -\begin{equation} - p_{B} = \frac{r_{s}^{2}}{2 r_{h}^{2} + r_{s}^{2}} -\end{equation} - -\begin{equation} - \frac{\partial p_{B}}{\partial \theta_{x}} - = \frac{-4 r_{s}^{2} r_{h}}{(2 r_{h}^{2} + r_{s}^{2})^{2}} - \frac{\partial r_{h}}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - r_{h} = | \underaccent{\tilde}{S} | - \frac{1}{\lambda} -\end{equation} - -\begin{equation} - r_{h} = (\underaccent{\tilde}{S} \cdot \underaccent{\tilde}{S})^{1 / 2} - - \frac{1}{\lambda} -\end{equation} - -\begin{equation} - \frac{\partial r_{h}}{\partial \theta_{x}} - = \frac{1}{2} - \frac{1}{\sqrt{\underaccent{\tilde}{S} \underaccent{\tilde}{S}}} - \frac{\partial \underaccent{\tilde}{S} \underaccent{\tilde}{S}} - {\partial \theta_{x}} -\end{equation} - -\begin{equation} - \frac{\partial \underaccent{\tilde}{S} \underaccent{\tilde}{S}} - {\partial \theta_{x}} - = 2 \underaccent{\tilde}{S} - \frac{\partial \underaccent{\tilde}{S}}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - \underaccent{\tilde}{S} = \underaccent{\tilde}{x} + S_{0} -\end{equation} - - -\begin{equation} - \frac{\partial \underaccent{\tilde}{S}}{\partial \theta_{x}} - = \frac{\partial \underaccent{\tilde}{x}}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - \underaccent{\tilde}{x} = A^{*} h -\end{equation} - -\begin{equation} - A^{*} = \mathbb{R}_{y} ( \mathbb{R}_{x} ( A_{0}^{*} ) ) -\end{equation} - -Aside: -\begin{equation} - \frac{\partial A h}{\partial u} - = A \frac{\partial h}{\partial u} + - \frac{\partial A}{\partial u} h -\end{equation} - -\begin{equation} - \frac{\partial \underaccent{\tilde}{x}}{\partial \theta_{x}} - = A^{*} \frac{\partial}{\partial \theta_{x}} \underaccent{\tilde}{h} + - \frac{\partial A^{*}}{\partial \theta_{x}} \underaccent{\tilde}{h} -\end{equation} -where $\partial \underaccent{\tilde}{h} / \partial \theta_{x} = 0$ and -$\partial A^{*} / \partial \theta_{x}$ corresponds to -\texttt{dA\_drotx = X}, $( \underaccent{\tilde}{h}, k, l )$ - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{x}} - = \mathbb{R}_{y} \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{x}} + - \frac{\partial R_{y}}{\partial \theta_{x}} R_{x} A_{0}^{*} -\end{equation} -where $\partial R_{y} / \partial \theta_{x} = 0$ - -\begin{equation} - \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{x}} - = R_{x} \frac{\partial A_{0}^{*}}{\partial \theta_{x}} + - \frac{\partial R_{x}}{\partial \theta_{x}} A_{0}^{*} -\end{equation} -where $\partial A_{0}^{*} / \partial \theta_{x} = 0$ and $\partial -R_{x} / \partial \theta_{x}$ in the code. - - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{y}} - = \mathbb{R}_{y} \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{y}} + - \frac{\partial R_{y}}{\partial \theta_{y}} (R_{x} A_{0}^{*}) -\end{equation} -where $\partial R_{x} A_{0}^{*} / \partial \theta_{y} = 0$ - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{y}} - = \frac{\partial R_{y}}{\partial \theta_{x}} (R_{x} A_{0}^{*}) -\end{equation} - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{x}} - = [R_{y}] \left[ \frac{\partial R_{x}}{\partial \theta_{x}} A_{0}^{*} \right] -\end{equation} - - - -\end{document} diff --git a/prime/whiteboard.tex b/prime/whiteboard.tex deleted file mode 100644 index f28acc34417..00000000000 --- a/prime/whiteboard.tex +++ /dev/null @@ -1,155 +0,0 @@ -\documentclass[12pt, letterpaper]{article} -\usepackage[english]{babel} -\usepackage{amsmath} -\usepackage{amssymb} -\usepackage{accents} - - %%%%%%%%%%%% - % PREAMBLE % - %%%%%%%%%%%% -\begin{document} -\selectlanguage{english} - -\section{Whiteboard 2013-09-10} - -\begin{equation} - f = \frac{1}{10} \sum_{i} \limits - ( G p I_{\mathrm{calc}} + K - I_{\mathrm{obs}})^{2} -\end{equation} -where $g_{i} = G p I_{\mathrm{calc}} + K - I_{\mathrm{obs}}$ is -\texttt{func\_callable} - -\begin{equation} - \frac{\partial g}{\partial \theta_{x}} - = ( G I_{\mathrm{c}} ) \frac{\partial p}{\partial \theta_{x}} -\end{equation} -where $\partial g / \partial \theta_{x}$ is \texttt{jacobian\_callable} - -\begin{equation} - \frac{\partial f}{\partial \theta_{x}} - = 2 \sum (G p I_{c} + K - I_{o}) (G I_{c}) - \frac{\partial p}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - \frac{\partial f}{\partial K} - = 2 \sum_{i} \limits g_{i} \frac{\partial g_{i}}{\partial K} - = 2 \sum_{i} \limits g_{i} -\end{equation} - -\begin{equation} - \frac{\partial f}{\partial G} - = 2 \sum_{i} \limits g_{i} \frac{\partial g_{i}}{\partial G} - = 2 \sum_{i} \limits g_{i} p I_{\mathrm{calc}} -\end{equation} - -\begin{equation} - \frac{\partial g_{i}}{\partial \theta_{x}}, - \frac{\partial g_{i}}{\partial \xi}, - \ldots -\end{equation} - -\begin{equation} - p_{B} = \frac{r_{s}^{2}}{2 r_{h}^{2} + r_{s}^{2}} -\end{equation} - -\begin{equation} - \frac{\partial p_{B}}{\partial \theta_{x}} - = \frac{-4 r_{s}^{2} r_{h}}{(2 r_{h}^{2} + r_{s}^{2})^{2}} - \frac{\partial r_{h}}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - r_{h} = | \underaccent{\tilde}{S} | - \frac{1}{\lambda} -\end{equation} - -\begin{equation} - r_{h} = (\underaccent{\tilde}{S} \cdot \underaccent{\tilde}{S})^{1 / 2} - - \frac{1}{\lambda} -\end{equation} - -\begin{equation} - \frac{\partial r_{h}}{\partial \theta_{x}} - = \frac{1}{2} - \frac{1}{\sqrt{\underaccent{\tilde}{S} \underaccent{\tilde}{S}}} - \frac{\partial \underaccent{\tilde}{S} \underaccent{\tilde}{S}} - {\partial \theta_{x}} -\end{equation} - -\begin{equation} - \frac{\partial \underaccent{\tilde}{S} \underaccent{\tilde}{S}} - {\partial \theta_{x}} - = 2 \underaccent{\tilde}{S} - \frac{\partial \underaccent{\tilde}{S}}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - \underaccent{\tilde}{S} = \underaccent{\tilde}{x} + S_{0} -\end{equation} - - -\begin{equation} - \frac{\partial \underaccent{\tilde}{S}}{\partial \theta_{x}} - = \frac{\partial \underaccent{\tilde}{x}}{\partial \theta_{x}} -\end{equation} - -\begin{equation} - \underaccent{\tilde}{x} = A^{*} h -\end{equation} - -\begin{equation} - A^{*} = \mathbb{R}_{y} ( \mathbb{R}_{x} ( A_{0}^{*} ) ) -\end{equation} - -Aside: -\begin{equation} - \frac{\partial A h}{\partial u} - = A \frac{\partial h}{\partial u} + - \frac{\partial A}{\partial u} h -\end{equation} - -\begin{equation} - \frac{\partial \underaccent{\tilde}{x}}{\partial \theta_{x}} - = A^{*} \frac{\partial}{\partial \theta_{x}} \underaccent{\tilde}{h} + - \frac{\partial A^{*}}{\partial \theta_{x}} \underaccent{\tilde}{h} -\end{equation} -where $\partial \underaccent{\tilde}{h} / \partial \theta_{x} = 0$ and -$\partial A^{*} / \partial \theta_{x}$ corresponds to -\texttt{dA\_drotx = X}, $( \underaccent{\tilde}{h}, k, l )$ - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{x}} - = \mathbb{R}_{y} \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{x}} + - \frac{\partial R_{y}}{\partial \theta_{x}} R_{x} A_{0}^{*} -\end{equation} -where $\partial R_{y} / \partial \theta_{x} = 0$ - -\begin{equation} - \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{x}} - = R_{x} \frac{\partial A_{0}^{*}}{\partial \theta_{x}} + - \frac{\partial R_{x}}{\partial \theta_{x}} A_{0}^{*} -\end{equation} -where $\partial A_{0}^{*} / \partial \theta_{x} = 0$ and $\partial -R_{x} / \partial \theta_{x}$ in the code. - - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{y}} - = \mathbb{R}_{y} \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{y}} + - \frac{\partial R_{y}}{\partial \theta_{y}} (R_{x} A_{0}^{*}) -\end{equation} -where $\partial R_{x} A_{0}^{*} / \partial \theta_{y} = 0$ - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{y}} - = \frac{\partial R_{y}}{\partial \theta_{x}} (R_{x} A_{0}^{*}) -\end{equation} - -\begin{equation} - \frac{\partial A^{*}}{\partial \theta_{x}} - = [R_{y}] \left[ \frac{\partial R_{x}}{\partial \theta_{x}} A_{0}^{*} \right] -\end{equation} - - - -\end{document}