diff --git a/scripts/oncotree_to_oncotree.py b/scripts/oncotree_to_oncotree.py new file mode 100644 index 00000000..d7250397 --- /dev/null +++ b/scripts/oncotree_to_oncotree.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 Memorial Sloan-Kettering Cancer Center. +# +# This library is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF +# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. The software and +# documentation provided hereunder is on an "as is" basis, and +# Memorial Sloan-Kettering Cancer Center +# has no obligations to provide maintenance, support, +# updates, enhancements or modifications. In no event shall +# Memorial Sloan-Kettering Cancer Center +# be liable to any party for direct, indirect, special, +# incidental or consequential damages, including lost profits, arising +# out of the use of this software and its documentation, even if +# Memorial Sloan-Kettering Cancer Center +# has been advised of the possibility of such damage. + +import argparse +import os +import sys +import urllib.request +import json + +ONCOTREE_WEBSITE_URL = "http://oncotree.mskcc.org/#/home?version=" +ONCOTREE_API_URL_BASE_DEFAULT = "http://oncotree.mskcc.org/api/" +ONCOTREE_VERSION_ENDPOINT = "versions" +ONCOTREE_TUMORTYPES_ENDPOINT = "tumorTypes" +VERSION_API_IDENTIFIER_FIELD = "api_identifier" +VERSION_RELEASE_DATE_FIELD = "release_date" +METADATA_HEADER_PREFIX = "#" +PASSTHROUGH_ONCOTREE_CODE_LIST = ["NA"] # These codes will be passed through the converter without examination or alteration +TOOL_VERSION_NUMBER = "1.2" + +# field names used for navigating tumor types +CHILDREN_CODES_FIELD = "children" +HISTORY_FIELD = "history" +ONCOTREE_CODE_FIELD = "code" +PARENT_CODE_FIELD = "parent" +PRECURSORS_FIELD = "precursors" +REVOCATIONS_FIELD = "revocations" + +# logging fields +GLOBAL_LOG_MAP = {} +CLOSEST_COMMON_PARENT_FIELD = "closest_common_parent" +CHOICES_FIELD = "choices" +NEIGHBORS_FIELD = "neighbors" +IS_LOGGED_FLAG = "logged" + +#-------------------------------------------------------------- +def fetch_oncotree_versions(oncotree_api_url_base): + # fetch available onctree versions from api + oncotree_version_endpoint_url = oncotree_api_url_base + ONCOTREE_VERSION_ENDPOINT + response = urllib.request.urlopen(oncotree_version_endpoint_url) + if response.getcode() != 200: + sys.stderr.write("ERROR (HttpStatusCode %d): Unable to retrieve OncoTree versions.\n" % (response.getcode())) + sys.exit(1) + return json.loads(response.read().decode("utf-8")) + +#-------------------------------------------------------------- +def validate_input_oncotree_versions(oncotree_versions_list, source_version, target_version): + valid_version_identifiers = [version[VERSION_API_IDENTIFIER_FIELD] for version in oncotree_versions_list] + if not source_version in valid_version_identifiers: + sys.stderr.write("ERROR: Source version (%s) is not a valid OncoTree version\n" % (source_version)) + sys.exit(1) + if not target_version in valid_version_identifiers: + sys.stderr.write("ERROR: Source version (%s) is not a valid OncoTree version\n" % (target_version)) + sys.exit(1) + +#-------------------------------------------------------------- +def validate_and_fetch_oncotree_version_release_dates(source_version, target_version, oncotree_api_url_base): + if source_version == target_version: + sys.stderr.write("Error: Source OncoTree version (%s) and target OncoTree version (%s) are the same. There is no need to convert this file.\n" % (source_version, target_version)) + oncotree_versions_list = fetch_oncotree_versions(oncotree_api_url_base) + + # validate source and target versions + validate_input_oncotree_versions(oncotree_versions_list, source_version, target_version) + + # return release dates of source and target versions from available OncoTree versions + source_oncotree_version_release_date = -1 + target_oncotree_version_release_date = -1 + for version in oncotree_versions_list: + if version[VERSION_API_IDENTIFIER_FIELD] == source_version: + source_oncotree_version_release_date = version[VERSION_RELEASE_DATE_FIELD] + if version[VERSION_API_IDENTIFIER_FIELD] == target_version: + target_oncotree_version_release_date = version[VERSION_RELEASE_DATE_FIELD] + return source_oncotree_version_release_date, target_oncotree_version_release_date + +#-------------------------------------------------------------- +def load_oncotree_version(oncotree_version_name, oncotree_api_url_base): + oncotree_nodes = {} + oncotree_tumortypes_endpoint = oncotree_api_url_base + ONCOTREE_TUMORTYPES_ENDPOINT + "?version=" + oncotree_version_name + response = urllib.request.urlopen(oncotree_tumortypes_endpoint) + if response.getcode() != 200: + sys.stderr.write("ERROR (HttpStatusCode %d): Unable to retrieve OncoTree version %s.\n" % (response.getcode(), oncotree_version_name)) + sys.exit(1) + for json_oncotree_node in json.loads(response.read().decode("utf-8")): + new_node = {} + new_node[PARENT_CODE_FIELD] = json_oncotree_node[PARENT_CODE_FIELD] + new_node[PRECURSORS_FIELD] = json_oncotree_node[PRECURSORS_FIELD] + new_node[REVOCATIONS_FIELD] = json_oncotree_node[REVOCATIONS_FIELD] + new_node[HISTORY_FIELD] = json_oncotree_node[HISTORY_FIELD] + new_node[ONCOTREE_CODE_FIELD] = json_oncotree_node[ONCOTREE_CODE_FIELD] + new_node[CHILDREN_CODES_FIELD] = [] + oncotree_nodes[json_oncotree_node[ONCOTREE_CODE_FIELD]] = new_node + # second pass, add in children + for oncotree_node in oncotree_nodes.values(): + try: + oncotree_nodes[oncotree_node[PARENT_CODE_FIELD]][CHILDREN_CODES_FIELD].append(oncotree_node[ONCOTREE_CODE_FIELD]) + except: + continue + return oncotree_nodes + +#-------------------------------------------------------------- +def get_header(file): + header = [] + with open(file, "r") as header_source: + for line in header_source: + if not line.startswith("#"): + header = line.rstrip().split('\t') + break + return header +#-------------------------------------------------------------- +# Takes a data_clinical_sample.txt file +# Saves header/commented lines as strings {row number: row} +# Additional processing to add ONCOTREE_CODE_OPTIONS column +def load_source_file(source_file): + header = get_header(source_file) + header_length = len(header) + headers_processed = False + source_file_mapped_list = [] + header_and_comment_lines = {} + header_line_number = 0 + + if "ONCOTREE_CODE" not in header: + sys.stderr.write("ERROR: Input file is missing column 'ONCOTREE_CODE'.\n") + sys.exit(1) + + with open(source_file, "r") as data_file: + for line_number, line in enumerate(data_file): + if '\r' in line: + sys.stderr.write("ERROR: source file (%s) is not in the required format (tab delimited, newline line breaks). carriage return characters encountered.\n") + sys.exit(1) + if line.startswith(METADATA_HEADER_PREFIX) or len(line.rstrip()) == 0: + header_and_comment_lines[line_number] = line + continue + if not headers_processed: + headers_processed = True + header_line_number = line_number + header_and_comment_lines[line_number] = line + continue + if len(line.split('\t')) != header_length: + sys.stderr.write("ERROR: Current row has a different number of columns than header row: %s\n" % line) + sys.exit(1) + data = dict(zip(header, map(str.strip, line.split('\t')))) + source_file_mapped_list.append(data) + + # This column has to be added after since zip was functioning on index + new_oncotree_code_index = header.index("ONCOTREE_CODE") + 1 + header.insert(new_oncotree_code_index, "ONCOTREE_CODE_OPTIONS") + + # add new column (ONCOTREE_CODE_OPTIONS) + for line_number in range(header_line_number): + header_and_comment_lines[line_number] = add_new_column(header_and_comment_lines[line_number], new_oncotree_code_index, "") + header_and_comment_lines[header_line_number] = add_new_column(header_and_comment_lines[header_line_number], new_oncotree_code_index, "ONCOTREE_CODE_OPTIONS") + # TODO: do the same thing for commented out lines inserted in random points throughout file (will have to make same changes for writing out records) + return source_file_mapped_list, header, header_and_comment_lines + +#-------------------------------------------------------------- +def add_new_column(row, new_column_index, column_name): + updated_row = row.split('\t') + # row is just a sentence (not a tab-delimited string), skip adding + if len(updated_row) < (new_column_index - 1): + return row + column_added_at_end = len(updated_row) == new_column_index + # column is being added at the end, add a return character and remove from previous last column + if column_added_at_end: + updated_row[new_column_index -1] = updated_row[new_column_index - 1].rstrip() + updated_row.insert(new_column_index, column_name + "\n") + else: + updated_row.insert(new_column_index, column_name) + return '\t'.join(updated_row) + +#-------------------------------------------------------------- +def remove_new_column(row, new_column_index): + updated_row = row.split('\t') + if len(updated_row) < new_column_index: + return row + column_removed_at_end = (len(updated_row) - 1) == new_column_index + # column was added at the end, remove new column and add return character back + if column_removed_at_end: + updated_row[new_column_index - 1] = updated_row[new_column_index - 1] + "\n" + del updated_row[new_column_index] + else: + del updated_row[new_column_index] + return '\t'.join(updated_row) + +#-------------------------------------------------------------- +# Uses a list of dictionaries, each dictionary represents a record/row +# Attempts to translate "ONCOTREE_CODE" value to target version equivalent +# Codes which map successfully (direct mapping w/o ambiguity or possible children) are placed in ONCOTREE_CODE column (ONCOTREE_CODE_OPTIONS empty) +# Codes which map ambiguously (no/possible mappings and/or new children) are placed in ONCOTREE_CODE_OPTIONS (ONCOTREE_CODE empty) +def translate_oncotree_codes(source_file_mapped_list, source_oncotree, target_oncotree, is_backwards_mapping): + for record in source_file_mapped_list: + source_oncotree_code = record["ONCOTREE_CODE"] + # initialize summary log for OncoTree code + if source_oncotree_code not in GLOBAL_LOG_MAP: + GLOBAL_LOG_MAP[source_oncotree_code] = { + NEIGHBORS_FIELD : [], + CHOICES_FIELD : [], + CLOSEST_COMMON_PARENT_FIELD : "", + IS_LOGGED_FLAG : False + } + translated_oncotree_code, is_easily_resolved = get_oncotree_code_options(source_oncotree_code, source_oncotree, target_oncotree, is_backwards_mapping) + if is_easily_resolved: + record["ONCOTREE_CODE"] = translated_oncotree_code + record["ONCOTREE_CODE_OPTIONS"] = "" + else: + record["ONCOTREE_CODE"] = "" + record["ONCOTREE_CODE_OPTIONS"] = translated_oncotree_code + return source_file_mapped_list + +#-------------------------------------------------------------- +# Given a "source" OncoTree code, return a string which can be in the following: +# 1) single code (single mapping, no children), True +# 2) single code but with children (single mapping, new children), False +# 3) multiple directly mapped options (w/ or w/o children), False +# 4) multiple related options (closest parents/children, don't include children), False +def get_oncotree_code_options(source_oncotree_code, source_oncotree, target_oncotree, is_backwards_mapping): + if source_oncotree_code in PASSTHROUGH_ONCOTREE_CODE_LIST: + return source_oncotree_code, True + if source_oncotree_code not in source_oncotree: + GLOBAL_LOG_MAP[source_oncotree_code][CHOICES_FIELD] = ["???"] + GLOBAL_LOG_MAP[source_oncotree_code][IS_LOGGED_FLAG] = True + if len(source_oncotree_code) == 0: + return "ONCOTREE_CODE column blank : use a valid OncoTree code or \"NA\"", False + else: + return ("%s -> ???, OncoTree code not in source OncoTree version" % (source_oncotree_code)), False + source_oncotree_node = source_oncotree[source_oncotree_code] + # get a set of possible codes that source code has been directly mapped to + possible_target_oncotree_codes = get_possible_target_oncotree_codes(source_oncotree_node, target_oncotree, is_backwards_mapping) + # resolve set of codes (cannot use possible_target_oncotree_nodes anymore) + target_oncotree_code, is_easily_resolved = resolve_possible_target_oncotree_codes(source_oncotree_code, possible_target_oncotree_codes, source_oncotree, target_oncotree, is_backwards_mapping) + return target_oncotree_code, is_easily_resolved + +#-------------------------------------------------------------- +# Given a "source" OncoTree code, return a set of directly mappable "target" OncoTree codes (though history, precursors, revocations) +# Rules for determining set diff based on mapping direction +def get_possible_target_oncotree_codes(source_oncotree_node, target_oncotree, is_backwards_mapping): + possible_target_oncotree_codes = set() + source_oncotree_code = source_oncotree_node[ONCOTREE_CODE_FIELD] + if is_backwards_mapping: + # Backwards mapping + # codes in history is in the target version (Same URI - different name) + possible_target_oncotree_codes.update(get_past_oncotree_codes_for_related_codes(source_oncotree_node, target_oncotree, HISTORY_FIELD)) + if not possible_target_oncotree_codes: # history overrides current code when history is present (e.g PTCLNOS) + if source_oncotree_code in target_oncotree: + possible_target_oncotree_codes.add(source_oncotree_code) + # codes in precusors is in the target version + possible_target_oncotree_codes.update(get_past_oncotree_codes_for_related_codes(source_oncotree_node, target_oncotree, PRECURSORS_FIELD)) + # skip checking codes in revocations - invalid codes which should not be considered + return possible_target_oncotree_codes + else: + # Forwards mapping + # codes where source code is in history (this should at most be 1 node - because its the same URI) + future_codes = get_future_related_oncotree_codes_for_source_code(source_oncotree_code, target_oncotree, HISTORY_FIELD) + if len(future_codes) > 1: + sys.stderr.write("ERROR: Future OncoTree has multiple codes with code %s in history\n" % (source_oncotree_code)) + sys.exit(1) + if len(future_codes) == 1: + possible_target_oncotree_codes.update(future_codes) + return possible_target_oncotree_codes + # codes where source code is in precursor (this can be more than 1, but can not intersect with revocations) + possible_target_oncotree_codes.update(get_future_related_oncotree_codes_for_source_code(source_oncotree_code, target_oncotree, PRECURSORS_FIELD)) + if len(possible_target_oncotree_codes) > 0: + return possible_target_oncotree_codes + # codes where source code is in revocations (this can be more than 1) + possible_target_oncotree_codes.update(get_future_related_oncotree_codes_for_source_code(source_oncotree_code, target_oncotree, REVOCATIONS_FIELD)) + if len(possible_target_oncotree_codes) > 0: + return possible_target_oncotree_codes + # at this point, no matches - check if source code exists in future OncoTree + if source_oncotree_code in target_oncotree: + possible_target_oncotree_codes.add(source_oncotree_code) + return possible_target_oncotree_codes + +#-------------------------------------------------------------- +# exclusively for mapping in backward direction +# looking through 'related nodes (history, precursors)', return if found in target (past) OncoTree +# i.e SLLCLL (SLL precusor, CLL precursor) -> SLL, CLL +def get_past_oncotree_codes_for_related_codes(source_oncotree_node, target_oncotree, field): + return [past_oncotree_code for past_oncotree_code in source_oncotree_node[field] if past_oncotree_code in target_oncotree] + +#-------------------------------------------------------------- +# exclusively for mapping in forward direction +# returns codes where source code is related in target (future) OncoTree +# i.e ALL -> BLL (ALL revocation), TLL (ALL revocation) +def get_future_related_oncotree_codes_for_source_code(source_oncotree_code, target_oncotree, field): + return [future_oncotree_code for future_oncotree_code, future_oncotree_node in target_oncotree.items() if source_oncotree_code in future_oncotree_node[field]] + +#-------------------------------------------------------------- +# Given a set of OncoTree codes, return a formatted string with following info (if available): +# 1a) (== 1 choice) directly mapped target OncoTree code +# 1b) (>1 choices) possible choices +# 1c) (0 choices) neighborhood/related codes +# 2) whether or not there are children +# +# * Presence of children not checked for cases with 0 choices +# since returned "options" are already in the general neighborhood +# * Additionally, if children are available, log the closest common parent for summary report +def resolve_possible_target_oncotree_codes(source_oncotree_code, possible_target_oncotree_codes, source_oncotree, target_oncotree, is_backwards_mapping): + number_of_new_children = 0 + # skip calculating number of children for case with no direct mappings + if len(possible_target_oncotree_codes) != 0: + number_of_new_children = get_number_of_new_children(source_oncotree_code, possible_target_oncotree_codes, source_oncotree, target_oncotree) + # is easily resolved if only one option with no children + is_easily_resolved = (len(possible_target_oncotree_codes) == 1 and not number_of_new_children) + + if is_easily_resolved: + oncotree_code_options = possible_target_oncotree_codes.pop() + if source_oncotree_code != oncotree_code_options and not GLOBAL_LOG_MAP[source_oncotree_code][IS_LOGGED_FLAG]: + GLOBAL_LOG_MAP[source_oncotree_code][CHOICES_FIELD].append(oncotree_code_options) + GLOBAL_LOG_MAP[source_oncotree_code][IS_LOGGED_FLAG] = True + # log if OncoTree code has changed + return oncotree_code_options, is_easily_resolved + + # case with 0 direct mappings, return neighborhood + log common parent + if len(possible_target_oncotree_codes) == 0: + neighboring_target_oncotree_codes = get_neighboring_target_oncotree_codes([source_oncotree_code], source_oncotree, target_oncotree, True, is_backwards_mapping) + oncotree_code_options = format_oncotree_code_options(source_oncotree_code, "Neighborhood: " + ','.join(neighboring_target_oncotree_codes), number_of_new_children) + closest_common_parent = get_closest_common_parent(neighboring_target_oncotree_codes, target_oncotree) + if not GLOBAL_LOG_MAP[source_oncotree_code][IS_LOGGED_FLAG]: + GLOBAL_LOG_MAP[source_oncotree_code][NEIGHBORS_FIELD].extend(neighboring_target_oncotree_codes) + GLOBAL_LOG_MAP[source_oncotree_code][CLOSEST_COMMON_PARENT_FIELD] = closest_common_parent + else: + ordered_possible_target_oncotree_codes = possible_target_oncotree_codes + if type(possible_target_oncotree_codes) is set: + ordered_possible_target_oncotree_codes = sorted(list(possible_target_oncotree_codes)) + oncotree_code_options = format_oncotree_code_options(source_oncotree_code, "{%s}" % ','.join(ordered_possible_target_oncotree_codes), number_of_new_children) + if not GLOBAL_LOG_MAP[source_oncotree_code][IS_LOGGED_FLAG]: + GLOBAL_LOG_MAP[source_oncotree_code][CHOICES_FIELD].extend(possible_target_oncotree_codes) + if number_of_new_children: + closest_common_parent = get_closest_common_parent(possible_target_oncotree_codes, target_oncotree) + GLOBAL_LOG_MAP[source_oncotree_code][CLOSEST_COMMON_PARENT_FIELD] = closest_common_parent + GLOBAL_LOG_MAP[source_oncotree_code][IS_LOGGED_FLAG] = True + return oncotree_code_options, is_easily_resolved + +#-------------------------------------------------------------- +def format_oncotree_code_options(source_oncotree_code, oncotree_code_options, number_of_new_children): + to_return = "%s -> %s" % (source_oncotree_code, oncotree_code_options) + if number_of_new_children: + to_return = to_return + ", more granular choices introduced" + return to_return + +#-------------------------------------------------------------- +# returns codes (in target version) which succesfully mapped from relatives +# will return self is source code is in target version +def get_neighboring_target_oncotree_codes(source_oncotree_codes, source_oncotree, target_oncotree, include_children, is_backwards_mapping): + immediate_relatives = set() + immediate_relatives_in_target_oncotree = set() + # collect codes which are directly related (parents + children) in first iteration, just parents in second+ iteration + for source_oncotree_code in source_oncotree_codes: + if source_oncotree[source_oncotree_code][PARENT_CODE_FIELD]: + immediate_relatives.add(source_oncotree[source_oncotree_code][PARENT_CODE_FIELD]) + if source_oncotree[source_oncotree_code][CHILDREN_CODES_FIELD] and include_children: + immediate_relatives.update(source_oncotree[source_oncotree_code][CHILDREN_CODES_FIELD]) + + for source_oncotree_code in source_oncotree_codes: + immediate_relatives_in_target_oncotree.update(get_possible_target_oncotree_codes(source_oncotree[source_oncotree_code], target_oncotree, is_backwards_mapping)) + + # no immediate relative could be mapped backwards - try again with expanded search + if not immediate_relatives_in_target_oncotree: + return get_neighboring_target_oncotree_codes(immediate_relatives, source_oncotree, target_oncotree, False, is_backwards_mapping) + else: # at least one code was mapped successfully backwards + return immediate_relatives_in_target_oncotree + +#-------------------------------------------------------------- +# Returns number of new children (number of target children codes not in list of source children codes) +def get_number_of_new_children(source_oncotree_code, possible_target_oncotree_codes, source_oncotree, target_oncotree): + children_in_source = [] + children_in_target = [] + children_in_source = set(get_children([source_oncotree_code], children_in_source, source_oncotree)) + children_in_target = set(get_children(possible_target_oncotree_codes, children_in_target, target_oncotree)) + number_of_new_children = len(children_in_target - children_in_source) + return number_of_new_children + +#-------------------------------------------------------------- +# Recusively builds of all children codes under a set of given onctoree codes +def get_children(oncotree_codes, all_children_codes, target_oncotree): + children = [] + for oncotree_code in oncotree_codes: + children.extend(target_oncotree[oncotree_code][CHILDREN_CODES_FIELD]) + if children: + all_children_codes.extend(children) + return get_children(children, all_children_codes, target_oncotree) + else: + return all_children_codes + +#-------------------------------------------------------------- +# get the common parent (furthest down the tree) for a set of oncotree_codes +def get_closest_common_parent(possible_target_oncotree_codes, target_oncotree): + oncotree_code_to_ancestors_mapping = {} + # for every possible target OncoTree code - construct an ordered list of ancestors + for oncotree_code in possible_target_oncotree_codes: + oncotree_code_ancestors = [oncotree_code] + oncotree_code_to_ancestors_mapping[oncotree_code] = get_ancestors(oncotree_code, oncotree_code_ancestors, target_oncotree) + min_length = min([len(ancestor_list) for ancestor_list in oncotree_code_to_ancestors_mapping.values()]) + # look across lists to find the earliest point where codes differ + closest_common_parent = get_earliest_common_parent(min_length, list(oncotree_code_to_ancestors_mapping.values())) + return closest_common_parent + +#-------------------------------------------------------------- +# used to construct list of ancestors - if parent exists, insert at beginning, and call again at a higher level +def get_ancestors(oncotree_code, oncotree_code_ancestors, target_oncotree): + if target_oncotree[oncotree_code][PARENT_CODE_FIELD]: + parent_oncotree_code = target_oncotree[oncotree_code][PARENT_CODE_FIELD] + oncotree_code_ancestors.insert(0, parent_oncotree_code) + return get_ancestors(parent_oncotree_code, oncotree_code_ancestors, target_oncotree) + else: + return oncotree_code_ancestors + +#-------------------------------------------------------------- +def get_earliest_common_parent(min_length, lists_of_all_ancestors): + # set reference code as the code at index (min_length - 1) for first list in lists_of_all_ancestors + oncotree_code = lists_of_all_ancestors[0][min_length - 1] + # skip first list in lists_of_all_ancestors + for list_of_ancestors in lists_of_all_ancestors[1:]: + if min_length > 0: + if list_of_ancestors[min_length - 1] != oncotree_code: + # at first mismatch, decrement min_length to move earlier in the list + min_length -= 1 + return get_earliest_common_parent(min_length, lists_of_all_ancestors) + return oncotree_code + +#-------------------------------------------------------------- +def write_to_target_file(translated_source_file_mapped_list, target_file, header, header_and_comment_lines): + all_easily_resolved = True + oncotree_code_options_index = header.index("ONCOTREE_CODE_OPTIONS") + for record in translated_source_file_mapped_list: + if record["ONCOTREE_CODE_OPTIONS"]: + all_easily_resolved = False + break + if all_easily_resolved: + header.remove("ONCOTREE_CODE_OPTIONS") + for line_number in range(len(header_and_comment_lines)): + header_and_comment_lines[line_number] = remove_new_column(header_and_comment_lines[line_number], oncotree_code_options_index) + + line_num = 0 + with open(target_file, "w") as f: + for record in translated_source_file_mapped_list: + while line_num in header_and_comment_lines: + f.write(header_and_comment_lines[line_num]) + line_num += 1 + formatted_data = map(lambda x: record.get(x,''), header) + f.write('\t'.join(formatted_data) + '\n') + line_num += 1 + sys.stderr.write("Primary target file written to %s\n" % (target_file)) + +#-------------------------------------------------------------- +# sorts logging map based on resolution type +# (e.g show unmappable nodes before ambiguous nodes +def sort_by_resolution_method(oncotree_code, logged_code): + # no direct mappings first + if logged_code[NEIGHBORS_FIELD]: + return "0" + oncotree_code + # has multiple possible choices and has children + elif len(logged_code[CHOICES_FIELD]) > 1 and logged_code[CLOSEST_COMMON_PARENT_FIELD]: + return "1" + oncotree_code + # has multiple possible choices and has no children + elif len(logged_code[CHOICES_FIELD]) > 1 and not logged_code[CLOSEST_COMMON_PARENT_FIELD]: + return "2" + oncotree_code + # has one choice and has children + elif len(logged_code[CHOICES_FIELD]) == 1 and logged_code[CLOSEST_COMMON_PARENT_FIELD]: + return "3" + oncotree_code + else: + return "4" + oncotree_code + +#-------------------------------------------------------------- +def write_summary_file(target_file, source_version, target_version): + oncotree_url = ONCOTREE_WEBSITE_URL + target_version + # Break logged nodes into subcategories and sort (alphabetically and resolution type) + # For each category, codes with more granular codes introduced are shown first + unmappable_codes = sorted([unmappable_code for unmappable_code, unmappable_node in GLOBAL_LOG_MAP.items() if unmappable_node[NEIGHBORS_FIELD]]) + ambiguous_codes = sorted([ambiguous_code for ambiguous_code, ambiguous_node in GLOBAL_LOG_MAP.items() if len(ambiguous_node[CHOICES_FIELD]) > 1], key = lambda k: sort_by_resolution_method(k, GLOBAL_LOG_MAP[k])) + partially_resolved_codes = sorted([resolved_code for resolved_code, resolved_node in GLOBAL_LOG_MAP.items() if len(resolved_node[CHOICES_FIELD]) == 1 and resolved_node[CLOSEST_COMMON_PARENT_FIELD] and ("???" not in resolved_node[CHOICES_FIELD])], key = lambda k: sort_by_resolution_method(k, GLOBAL_LOG_MAP[k])) + completely_resolved_codes = sorted([resolved_code for resolved_code, resolved_node in GLOBAL_LOG_MAP.items() if len(resolved_node[CHOICES_FIELD]) == 1 and not resolved_node[CLOSEST_COMMON_PARENT_FIELD] and ("???" not in resolved_node[CHOICES_FIELD])]) + unrecognized_codes = sorted([unrecognized_code for unrecognized_code, unrecognized_node in GLOBAL_LOG_MAP.items() if ("???" in unrecognized_node[CHOICES_FIELD]) ], key = lambda k: sort_by_resolution_method(k, GLOBAL_LOG_MAP[k])) + + html_summary_file = os.path.splitext(target_file)[0] + "_summary.html" + with open(html_summary_file, "w") as f: + # General info + f.write("\n\n\nMapping Summary\n\n\n\n") + f.write("

Mapping Summary

\n") + f.write("

Tool version: v.%s
" % (TOOL_VERSION_NUMBER)) + f.write("Mapped %s to %s
" % (source_version, target_version)) + f.write("All resolutions should be made with version: %s\n" % (oncotree_url, target_version)) + f.write("

Contents

 \n \n") + # Unrecognized codes - action required, but not guidance. Just list them + if unrecognized_codes: + f.write("

The following codes were not present in the source OncoTree version:

\n") + for oncotree_code in unrecognized_codes: + f.write("

Original Code: %s
\n" % ("<blank>" if len(oncotree_code) == 0 else oncotree_code)) + f.write("New Code cannot be determined\n") + # Unmappable codes - printed first since they MUST be resolved with manual tree exploration + if unmappable_codes: + f.write("

The following codes could not be mapped to a code in the target version and require additional resolution:

\n") + for oncotree_code in unmappable_codes: + f.write("

Original Code: %s
\n" % (oncotree_code)) + f.write("Closest Neighbors: %s
\n" % ','.join(GLOBAL_LOG_MAP[oncotree_code][NEIGHBORS_FIELD])) + f.write("To resolve, please refer to closest shared parent node %s and its descendants here

\n" % ((GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD]), (oncotree_url + "&search_term=(" + GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD] + ")"))) + # Ambiguous codes - printed second since they MUST be resolved but already provide choices + if ambiguous_codes: + f.write("

The following codes mapped to multiple codes in the target version. Please select from provided choices:

\n") + for oncotree_code in ambiguous_codes: + f.write("

Original Code: %s
\n" % (oncotree_code)) + f.write("Choices: %s
\n" % ','.join(GLOBAL_LOG_MAP[oncotree_code][CHOICES_FIELD])) + if GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD]: + f.write("*Warning: Target version has introduced more granular nodes.
\n") + f.write("You can examine the closest shared parent node %s and its descendants here

\n" % ((GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD]), (oncotree_url + "&search_term=(" + GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD] + ")"))) + # Directly mapped codes - no action required, might want to explore more granular choices + if partially_resolved_codes: + f.write("

The following codes mapped to exactly one code but more granular codes have been introduced:

\n") + for oncotree_code in partially_resolved_codes: + f.write("

Original Code: %s
\n" % (oncotree_code)) + f.write("New Code: %s
\n" % ','.join(GLOBAL_LOG_MAP[oncotree_code][CHOICES_FIELD])) + if GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD]: + f.write("*Warning: Target version has introduced more granular nodes.
\n") + f.write("You can examine the closest shared parent node %s and its descendants here

\n" % ((GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD]), (oncotree_url + "&search_term=(" + GLOBAL_LOG_MAP[oncotree_code][CLOSEST_COMMON_PARENT_FIELD] + ")"))) + # Directly mapped codes - no action required, might want to explore more granular choices + if completely_resolved_codes: + f.write("

The following codes mapped to exactly one code:

\n") + for oncotree_code in completely_resolved_codes: + f.write("

Original Code: %s
\n" % (oncotree_code)) + f.write("New Code: %s
\n" % ','.join(GLOBAL_LOG_MAP[oncotree_code][CHOICES_FIELD])) + sys.stderr.write("Mapping summary HTML file written out to %s\n" % (html_summary_file)) + +def usage(parser, message): + if message: + sys.stderr.write("%s\n" % (message)) + sys.stderr.write("%s\n" % (parser.print_help())) + sys.exit(1) + +#-------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-a", "--auto-mapping-enabled", help = "enable automatic resolution of ambiguous mappings", action = "store_true") + parser.add_argument("-i", "--source-file", help = "source file provided by user", required = True) + parser.add_argument("-o", "--target-file", help = "destination file to write out new file contents", required = True) + parser.add_argument("-s", "--source-version", help = "current OncoTree version used in the source file", required = True) + parser.add_argument("-t", "--target-version", help = "OncoTree version to be mapped to in the destination file", required = True) + parser.add_argument("-u", "--oncotree-url", required = False, help=argparse.SUPPRESS) + args = parser.parse_args() + + source_file = args.source_file + target_file = args.target_file + source_version = args.source_version + target_version = args.target_version + oncotree_url = args.oncotree_url + + oncotree_api_url_base = ONCOTREE_API_URL_BASE_DEFAULT + if oncotree_url: + oncotree_api_url_base = oncotree_url + + if not source_file or not target_file or not source_version or not target_version: + usage(parse, "Error: missing arguments") + + if not os.path.isfile(source_file): + sys.stderr.write("Error: cannot access source file (%s) : no such file\n" % (source_file)) + sys.exit(1) + + source_oncotree_version_release_date, target_oncotree_version_release_date = validate_and_fetch_oncotree_version_release_dates(source_version, target_version, oncotree_api_url_base) + is_backwards_mapping = target_oncotree_version_release_date < source_oncotree_version_release_date # determines directionality of source - target OncoTree mapping + source_file_mapped_list, header, header_and_comment_lines = load_source_file(source_file) + source_oncotree = load_oncotree_version(source_version, oncotree_api_url_base) + target_oncotree = load_oncotree_version(target_version, oncotree_api_url_base) + translated_source_file_mapped_list = translate_oncotree_codes(source_file_mapped_list, source_oncotree, target_oncotree, is_backwards_mapping) + write_to_target_file(translated_source_file_mapped_list, target_file, header, header_and_comment_lines) + write_summary_file(target_file, source_version, target_version) + sys.stderr.write("OncoTree version conversion completed.\n") + +if __name__ == '__main__': + main() diff --git a/scripts/ontology_to_ontology_mapping_tool/README.md b/scripts/ontology_to_ontology_mapping_tool/README.md new file mode 100644 index 00000000..cbee90e2 --- /dev/null +++ b/scripts/ontology_to_ontology_mapping_tool/README.md @@ -0,0 +1,29 @@ +## Ontology to Ontology Mapping Tool + +The Ontology Mapping tool was developed to facilitate the mapping between different cancer classification systems. We currently support mapping between OncoTree, ICD-O, NCIt, UMLS and HemeOnc systems. + +### Prerequisites +The Ontology Mapping tool runs on python 3 and requires `pandas` and `requests` libraries. These libraries can be installed using +``` +pip3 install pandas +pip3 install requests + ``` + +### Running the tool + +The tool can be run with the following command: +``` +python --source-file --target-file --source-code --target-code +``` + +**Options** +``` + -i | --source-file: This is the source file path. The source file must contain one of the ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE in the file header and it must contain codes corresponding to the Ontology System. + -o | --target-file: This is the path to the target file that will be generated. It will contain ontologies mapped from source code in to . + -s | --source-code: This is the source ontology code in . It must be one of the ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE. + -t | --target-code: This is the target ontology code that the script will attempt to map the source file ontology code to. It must be one of the ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE. +``` + +**Note** +- The source file should be tab delimited and should contain one of the ontology: ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE in the file header. +- We currently are allowing only one ontology to another ontology mapping. In the future, we plan to extend the tool to support mapping to multiple ontology systems. diff --git a/scripts/ontology_to_ontology_mapping_tool/file_comparison/README.md b/scripts/ontology_to_ontology_mapping_tool/file_comparison/README.md new file mode 100644 index 00000000..bf6c33e9 --- /dev/null +++ b/scripts/ontology_to_ontology_mapping_tool/file_comparison/README.md @@ -0,0 +1,7 @@ +## Ontology to Ontology Mapping Tool + +This tool helps compare existing OncoTree mapped ontologies against a reference OncoTree Codes file, and gives an output file named "missing_oncotree_codes.xlsx" that consist of a list of OncoTree codes that are missing from the inputted file. In order to use this tool, you must: +1. Download the 'refrence_oncotree_codes.xlsx' file +2. Input an excel file that consists of OncoTree codes in column 1 and other ontology codes (UMLS, NCIT, SNOMED, ICDO Topography, HEMEONC, or ICDO Morphology) in the other columns. + +This tool MUST consist of OncoTree codes in column 1 to compare the two files. The output file will only give OncoTree codes that are missing from the file you inputted. diff --git a/scripts/ontology_to_ontology_mapping_tool/file_comparison/file_comparison.py b/scripts/ontology_to_ontology_mapping_tool/file_comparison/file_comparison.py new file mode 100644 index 00000000..07e5b338 --- /dev/null +++ b/scripts/ontology_to_ontology_mapping_tool/file_comparison/file_comparison.py @@ -0,0 +1,29 @@ +import pandas as pd +import os + +file_a = pd.read_excel("reference_oncotree_codes.xlsx") +oncotree_codes_a = file_a.iloc[:, 0].dropna() + +while True: + file_b = input("Enter name of the file in an Excel format ('.xlsx')") + if '.xlsx' not in file_b: + print("You forgot to add '.xlsx'. Please try again and add the required '.xlsx'.") + elif '.xlsx' in file_b: + if os.path.exists(file_b): + file_b = pd.read_excel(file_b) + file_b = file_b.astype(str) + file_b.replace({'': pd.NA}, inplace=True) + oncotree_codes_b = file_b.iloc[:, 0].dropna() + + missing_codes = oncotree_codes_a[~oncotree_codes_a.isin(oncotree_codes_b)] + + print("The following OncoTree codes from File A are NOT found in File B:") + for code in missing_codes: + print("-", code) + + + output_df = missing_codes + output_file_path = "missing_oncotree_codes.xlsx" + output_df.to_excel(output_file_path, index=False) + else: + print("This file does not exist in your folders. Try transferring the excel file into the same folder as where this code is in") diff --git a/scripts/ontology_to_ontology_mapping_tool/ontology_mappings.txt b/scripts/ontology_to_ontology_mapping_tool/ontology_mappings.txt new file mode 100644 index 00000000..2eaf161c --- /dev/null +++ b/scripts/ontology_to_ontology_mapping_tool/ontology_mappings.txt @@ -0,0 +1,853 @@ +ONCOTREE_CODE NCIT_CODE UMLS_CODE ICDO_TOPOGRAPHY_CODE ICDO_MORPHOLOGY_CODE HEMEONC_CODE +MMB C3706 C0205833 +AIS C4123 C0334276 C80.9 8140/2 +AASTR C9477 C0334579 C72.9 9401/3 +FL C3209 C0024301 C42.4 9690/3 599 +VIMT C4286 C0334520 C57.9 9080/3 +KIDNEY C12415 C0022646 +MDEP C4327 C0334596 C72.9 9501/3 +PAOS C8969 C0206642 C41.9 9192/3 +PRSCC C6766 C1300585 C61.9 8002/3 +DSTAD C9159 C0279635 C16.9 8145/3 +SECOS C53704 C1710042 C41.9 9184/3 +ARMS C3749 C0206655 C49.9 8920/3 +PT C7575 C0238031 C50.9 9020/1 +MSCHW C6970 C1306247 C47.9 9560/1 +SCST C4862 C0600113 C56.9 8590/1 +MBC C5164 C1334708 C50.9 8575/3 +SCCE C7982 C0279674 C53.9 8002/3 +AWDNET C96422 C3272767 C18.1 8240/3 +ROCY C4526 C0346255 C64.9 8290/0 +VMM C27394 C2004576 C57.9 8746/3 +LAM C38153 C0349649 C34.9 9174/1 +CHDM C2947 C0008487 C41.9 9370/3 +ACPP C53686 C1266176 C72.9 9390/1 +PSTAD C5472 C1333785 C16.9 8260/3 +MEL C3224 C0025202 C44.9 8720/3 629 +CABC C40213 C1516403 +SCCO C27390 C2212006 C56.9 8041/3 +MFS C6496 C0334454 C49.9 8811/3 +GNBL C3790 C0206718 C47.9 9490/3 +CM C4550 C0346360 C69.0 8720/3 +PRCC C6975 C1306837 C64.9 8260/3 +BLPT C5316 C1332592 +SELT C4944 C0748616 C72.9 8000/3 +LUPC C45542 C1711397 C34.9 8022/3 +VMGCT C4290 C0334524 C57.9 9085/3 +GCLC C4452 C0345960 C34.9 8031/3 +UASC C4519 C0346202 C55.9 8560/3 +PXA C4323 C0334586 C72.9 9424/3 +EPM C3017 C0014474 C72.9 9391/3 +OFMT C6582 C1266128 C49.9 8842/0 +PACT C41247 C1518872 C25.9 8000/3 +LUSC C3493 C0149782 C34.9 8070/3 +FLC C4131 C0334287 C22.0 8171/3 +CMC C60641 C1880119 C18.9 8510/3 +SRAP C43554 C1711320 C18.1 8490/3 +ATM C4723 C0431122 C70.9 9539/1 +MASCC C40358 C1519487 +CHOM C6908 C1370510 +LUNG C12468 C0024109 C34.9 8000/3 46104 +PCNSL C9301 C0280803 C42.4 9680/3 +LXSC C4044 C0280324 C32.9 8070/3 +ADRENAL_GLAND C12666 C0001625 C74.9 8000/3 +RHM C6909 C0259786 C70.9 9538/3 +WDLS C4250 C1370889 C49.9 8851/3 +PEMESO C7633 C1377610 C48.2 9050/3 +HNSC C34447 C1168401 C76.0 8070/3 +AFX C4246 C0346053 +DESM C37257 C0334439 C44.9 8745/3 +MBOV C40036 C0279664 C56.9 8472/1 +OEC C8108 C0346183 +SCLC C4917 C0149925 C34.9 8041/3 666 +PMA C40315 C1519086 C72.9 9425/3 +PSCC C7729 C0238348 C60.9 8070/3 +ACBC C5130 C1332167 C50.9 8200/3 +CEAIS C4520 C0346203 C53.9 8140/2 +ODYS C8106 C0346185 C56.9 9060/3 +OPHSC C8181 CL497390 C10.9 8070/3 643 +PD C3301 C1704323 C50.0 8540/3 +SUBE C3795 C0206725 C72.9 9383/1 +ARMM C4639 C0349538 C21.8 8746/3 +TISSUE C12801 C0040300 +FIOS C4020 C0279602 C41.9 9182/3 +HGSOC C56.9 8441/3 +THHC C4946 C0749424 C73.9 8290/3 +LUNE C6875 C1265996 C34.9 8013/3 +THME C3879 C0238462 C73.9 8510/3 +IDC C4194 C1134719 C50.9 8521/3 +TSTAD C5473 C1333791 C16.9 8211/3 +OGBL C39985 C1518716 +PSC C5712 C1335316 C25.9 8441/0 +RSCC C116317 C2212425 C64.9 8002/3 +BLCA C39851 C0279680 C67.9 8120/3 569 +BCCA C2948 C0008497 C72.9 9100/3 +NMZL C8863 C0855139 +SWDNET C95871 C3272399 C16.9 8240/3 +USCC C6165 C1336890 C68.0 8070/3 +EMCHS C27502 C1275278 C41.9 9231/3 +SRCBC C39823 C1512742 C67.9 8490/3 +ONBL C3789 C0206717 C72.9 9522/3 +AMPCA C3908 C0262401 C24.1 8010/3 651 +CCS C3745 C0206651 C49.9 9044/3 +IMTL C39740 C1518038 C34.9 8827/1 +MFH C4247 C0334463 C49.9 8830/3 +FIBS C3043 C0016057 C49.9 8810/3 +COM C47848 C1711312 +BMGCT C4290 C0334524 C71.9 9085/3 +SCHW C3269 C0027809 C47.9 9560/0 +CERVIX C12311 C0007874 +SIC C7724 C0238196 C17.9 8010/3 +NSCLC C2926 C0007131 C34.9 8046/3 642 +PBS C4670 C0349667 C50.9 8800/3 +UMC C40144 C0854923 C55.9 8480/3 +VMT C9015 C1368910 C57.9 9080/0 +OGCT C3873 C0238324 C56.9 9064/3 +MPC C50401 C1302808 +CEVG C40208 C1516425 +ASTB C4324 C0334587 C72.9 9430/3 +MAC C7581 C0346027 +VDYS C8106 C0346185 C57.9 9060/3 +PHCH C96804 C3273047 C24.9 8160/3 +ICEMU C40203 C1516422 C53.9 8480/3 +DDCHS C6476 C0862878 C41.9 9243/3 +SACA C9272 C0948750 C08.9 8010/3 +GMN C3753 C0206660 C72.9 9064/3 +PLRMS C4258 C0334480 C49.9 8901/3 +PANCREAS C12393 C0030274 C25.9 8000/3 +OM C8562 C0558356 C69.9 8720/3 +BPSCC C6980 C1332462 C60.9 8083/3 +OSMBT C40038 C1511264 C56.9 8442/1 +RWDNET C96159 C3272610 C20.9 8240/3 +CCOC C54300 C0475829 C41.0 9341/3 +LCH C3107 C0019621 C42.4 9752/1 +LNET C5670 C1334452 C34.9 8240/3 +LUACC C5666 C1334439 C34.9 8200/3 +PCNSM C5505 C1332888 C72.9 8720/3 577 +MAAP C43558 C1706832 C18.1 8480/3 +ASPS C3750 C0206657 C49.9 9581/3 +PMHE C121668 C3840252 +PRNET C3787 C0206715 C72.9 8010/3 +STOMACH C12391 C0038351 C16.9 8000/3 38787 +BA C5184 C1332614 C50.9 9120/3 +AMPULLA_OF_VATER C13011 C0042425 +SKIN C12470 C1123023 C44.9 8000/3 46102 +BLADDER C12414 C0005682 C67.9 8000/3 +TCCA C7733 C0238449 C62.9 9100/3 +MMBL C9497 C1275668 +MUCC C3772 C0206694 C08.9 8430/3 +USTUMP C40177 C1519864 C55.9 8897/1 +USARC C6339 C0338113 C55.9 8800/3 +ECAD C28327 C1299237 C53.0 8140/3 +AMOL C4861 C0023465 C42.4 9891/3 +SOC C7550 C1335177 C56.9 8441/3 +OCS C9192 C0392998 C56.9 8980/3 +BLSC C4031 C0279681 C67.9 8070/3 +CHL C7164 C1333064 C42.4 9650/3 +MBL C3222 C0025149 C72.9 9470/3 628 +NHL C3211 C0024305 C42.4 9590/3 46089 +DSRCT C8300 C0281508 C49.9 8806/3 +ALUCA C45551 C1708766 C34.9 8249/3 +SPC C6870 C1336027 C50.9 8050/3 +CHBL C2945 C0008441 C41.9 9230/0 +EBOV C7983 C0334338 C56.9 8380/1 +THPD C6040 C1266050 C73.9 8337/3 +ES C4817 C0553580 C41.9 9260/3 597 +SGO C5932 C1335906 +UCP C5722 C1336861 C25.9 8020/3 +MRC C7572 C4049328 C64.9 8510/3 +CSCHW C4724 C0431124 C47.9 9560/0 +GBC C3844 C0235782 C23.9 8000/3 600 +DIA C9476 C0457179 +PBL C9344 C0205898 C75.3 9362/3 +EVN C92555 C2985175 C72.9 9506/1 +OVARY C12404 C0029939 C56.9 8000/3 645 +PTCA C4536 C0346300 C75.1 8272/3 +USC C27838 C0854924 C55.9 8441/3 +PERITONEUM C12770 C0031153 C48.2 8000/3 +SNA C160976 CL970005 C30.0 8140/3 +WDTC C7153 C1337013 C73.9 8010/3 +TSCST C3794 C0206724 C62.9 8590/1 +HDCS C27349 C0334663 C49.9 9757/3 +MASC C40361 C1510796 +AOAST C6959 C0431108 C72.9 9382/3 +MDS C3247 C3463824 C42.4 9989/3 634 +APAD C7718 C0238003 C18.1 8140/3 +LIVER C12392 C0023884 C22.0 8000/3 +CSCC C4819 C0553723 C44.9 8070/3 588 +EMYOCA C4199 C0334392 C76.0 8562/3 +LUAS C9133 C0279557 C34.9 8560/3 +CEMN C40254 C1516419 C53.9 9110/3 +BPDCN C7203 C1301363 C42.4 9727/3 570 +SBOV C5226 C1332598 C56.9 8442/1 +BLCLC C7266 C1332463 C34.9 8012/3 +BIMT C7014 C1332883 C71.9 9080/3 +EMBCA C6341 C0238448 C62.9 9070/3 +ADPA C27534 C1367789 +LGSOC C105556 C3642255 C56.9 8460/3 +CDRCC C6194 C1266044 C64.9 8319/3 +AITL C7528 C0020981 C42.4 9705/3 +SEBA C40310 C0206684 C44.9 8410/3 +LGCOS C6474 C1266163 C41.9 9187/3 +CHM C4871 C0678213 C55.9 9100/0 +ISTAD C9157 C0279633 C16.9 8140/3 +MCHS C53493 C1708980 C41.9 9240/3 +NPC C3871 C2931822 C11.9 8010/3 639 +LGESS C4263 C0334486 C54.1 8931/3 +SCEMU C40205 C1516424 +MBT C6974 C1334936 C71.9 8000/1 +UTUC C7716 C0220648 C68.9 8120/3 +PPCT C44.9 8000/1 +OMGCT C8114 C0280135 C56.9 9085/3 +OPE C39990 C1514199 +STSC C6764 C1333788 C16.9 8002/3 +NBL C3270 C0027819 C47.9 9500/3 640 +UA C39843 C1511204 C67.7 8140/3 +SCB C39824 C1512743 C67.9 8033/3 +CEAD C4029 C0279672 C53.9 8140/3 579 +HEMA C3085 C0018916 C49.9 9120/0 +EPIS C3714 C0205944 C49.9 8804/3 53861 +SCOAH C94537 C2986561 C75.1 8290/0 +MCHSCNS C3737 C0206637 +PLLS C3705 C0205825 C49.9 8854/3 +GCTSTM C4289 C0334523 C62.9 9084/3 +BTOV C39954 CL323981 C56.9 9000/0 +DDLS C3704 C0205824 C49.9 8858/3 +ETANTR C4915 C0700367 +BPT C5196 C1332533 +LUMEC C45544 C1708778 C34.9 8430/3 +SAAD C8021 C0279746 C08.9 8140/3 +MDLC C5160 CL007210 C50.9 8522/3 +OVT C4381 C0341823 C56.9 8010/3 +RGNT C67559 C2347979 C71.7 9509/1 +NSGCT C9313 C1336724 C62.9 9065/3 +OYST C8107 C0346188 C56.9 9071/3 +SPN C37212 C1336030 C25.9 8452/1 +WPSCC C6981 C1337009 +UAS C6336 C1336917 C55.9 8933/3 +LGFMS C45202 C1275282 C49.9 8840/3 +SDCA C5904 C1301194 C08.9 8500/3 +AGA C5609 C1266027 C21.0 8215/3 +BRSRCC C5175 C1335964 C50.9 8490/3 +TYST C8000 C0279708 C62.9 9071/3 +PAAD C8294 C0281361 C25.9 8140/3 648 +MGCT C6347 C1336720 C62.9 9085/3 +PB C4265 C0334489 C25.9 8971/3 +CECC C6344 C1332912 C53.9 8310/3 +LMS C3158 C0023269 C49.9 8890/3 54053 +SM C9235 C0221013 C42.4 9741/3 669 +SEM C9309 C0036631 C62.9 9061/3 +STAD C4004 C0278701 C16.9 8140/3 601 +DTE C27524 C0432526 +TESTIS C12412 C0039597 C62.9 8000/3 671 +POCA C5560 C1266065 C44.9 8409/3 +SGAD C3682 C1883403 C44.9 8400/3 +SCBC C9461 C1332564 C67.9 8041/3 +LCIS C4018 C0279563 C50.9 8520/2 +DIG C4738 C1321878 C72.9 8000/3 +DMBL C4956 C0751291 C72.9 9471/3 +ANSC C9161 C1412036 C21.0 8070/3 558 +UTERUS C12405 C0042149 C55.9 8000/3 +OS C9145 C0029463 C41.9 9180/3 644 +MYCF C3246 C0026948 C42.4 9700/3 586 +SARCL C45540 C1708781 C34.9 8033/3 +IMTB C6177 C1336891 C67.9 8825/1 +ODG C3288 C0751396 C72.9 9450/3 +ESS C8973 C0206630 C54.1 8930/3 +CCHM C47847 C1707042 C50.9 8575/3 +NFIB C3272 C0027830 C47.9 9540/0 +PEL C6915 C1292753 C42.4 9678/3 +MRLS C27781 C0206634 C49.9 8852/3 +GCTB C121932 C0206638 C41.9 9250/1 53894 +PINC C6966 C0917890 +GS C4221 C1266111 C49.9 8710/3 +MT C9305 C0006826 C71.9 8000/3 +GIST C3868 C0238198 C26.9 8936/1 602 +BMGT C7015 C1336704 +BRAIN C12438 C3714787 C72.9 8000/3 46090 +AML C3171 C0023467 C42.4 9840/3 552 +SPIR C4170 C0334347 C44.9 8403/0 +CCM C4722 C0431121 C72.9 9538/1 +UCA C9106 C0700101 C68.0 8000/3 +THYROID C12400 C0040132 C73.9 8000/3 675 +MPT C4504 C0346154 C50.9 9020/3 +PSTT C3757 C0206666 +LECLC C45519 C1708792 C34.9 8082/3 +PPB C5669 C1266144 C34.9 8973/3 +OSMCA C40090 C0279392 C56.9 8474/3 +BFN C40405 C1511309 +RMS C3359 C0035412 C49.9 8900/3 661 +THPA C4035 C0238463 C73.9 8260/3 +CMML C3178 C0023480 C42.4 9945/3 583 +SCOS C4023 C0279622 C41.9 9185/3 +CHRCC C4146 C1266042 C64.9 8317/3 +PNS C12465 C0206417 +AA C6936 C1306242 +ANM C4051 C0259785 C70.9 9530/3 +ATRT C6906 C1266184 C72.9 9508/3 +UEC C6287 C1336905 C55.9 8380/3 +PRNE C5545 C1335515 C61.9 8246/3 +VMA C40252 C1519925 +ASTR C60781 C0004114 C72.9 9400/3 +GB C3058 C0017636 C72.9 9440/3 +VA C7981 C0279668 C52.9 8140/3 +GNG C3788 C0206716 C72.9 9505/1 +MOV C5242 C1335168 C56.9 8480/3 +GNC C6934 CL378224 C72.9 9490/0 +APE C4049 C0280788 C72.9 9392/3 +GSARC C3796 C0206726 C72.9 9442/3 +FA C3744 C0206650 +BLAD C4032 C0279682 C67.9 8140/3 +EOV C7979 C0346163 C56.9 8380/3 +RBL C7541 C0035335 C69.2 9510/3 +STMYEC C7596 C0334699 C49.9 8982/3 +ANGL C92552 C2363903 C72.9 9431/1 +AGNG C4717 C0431112 C72.9 9505/3 +MSCC C5177 C1336079 C50.9 8070/3 +SOFT_TISSUE C12471 C0225317 C49.9 8000/3 667 +NST C4972 C0206727 C47.9 9540/3 +CCRCC C4033 C0279702 C64.9 8310/3 +CEAS C4519 C0346202 C53.9 8560/3 +UUC C6345 C0850327 C55.9 8020/3 +THYMUS C12433 C0040113 +UCEC C7558 C0476089 C54.1 8010/3 593 +CSCLC C9137 C0334240 C34.9 8045/3 +SMZL C4663 C0349632 C42.4 9689/3 +BMT C7013 C1332886 C71.9 9080/0 +UCCA C27246 C0279677 C55.9 9100/3 +MPN C4345 C1292778 C42.4 9960/3 46095 +PHM C4293 C0334529 +USTAD C5476 C1336858 C16.9 8140/3 +CPC C4715 C0431109 C72.9 9390/3 +SKCM C3510 C0151779 C44.9 8720/3 +ACC C9325 C0206686 C74.9 8370/3 555 +MLYM C3208 C0024299 C71.9 9590/3 +IDCS C9282 C1260326 C49.9 9757/3 +SRCC C27893 C1266043 C64.9 8318/3 +BL C2912 C0006413 C42.4 9687/3 573 +BRAME C6899 C1510795 C50.9 8983/0 +TRCC C27891 C1337036 C64.9 8312/3 +SCSRMS C121654 C4053999 C49.9 8912/3 +PAAC C7977 C0279661 C25.9 8550/3 +CACC C6346 C1332911 C53.9 8200/3 +PINT C3328 C1412004 C75.3 9360/1 +IHM C6985 C0008493 +SRCCR C9168,C7967 C0279654,C1707436 C18.9 8490/3 +MCL C4337 C0334634 C42.4 9673/3 625 +ACA C9003 C0206667 C74.9 8370/0 +PLEURA C12469 C0032225 +OIMT C8111 C0346182 C56.9 9080/3 +VGCE C40208 C1516425 C53.9 8262/3 +IAMPCA C27415 C1332247 C24.1 8144/3 +UELMS C40174 C1519851 C55.9 8891/3 +PRSC C5536 C1302530 C61.9 8070/3 +CESC C4028 C0279671 C53.9 8070/3 +PPTID C6967 C1367859 C75.3 9362/3 +CCOV C40076 C0346164 C56.9 8310/3 +SYNS C3400 C0039101 C49.9 9040/3 +ERMS C8971 C0206656 C49.9 8910/3 +EYE C12401 C0015392 +CUP C3812 C0220647 C80.9 8000/3 +UM C7712 C0220633 C69.4 8720/3 677 +RAML C3888 C0241961 C64.9 8860/0 +WM C80307 C0024419 C42.4 9761/3 682 +CHOL C4436 C0206698 C22.0 8160/3 580 +OTHER C17649 C0205394 C80.9 8000/3 +OCSC C4833 C0585362 C14.8 8070/3 +ANGS C3088 C0018923 C49.9 9120/3 +GEJ C9296 C1332166 C15.9 8140/3 +BOWEL C12736 C0021853 C26.0 8000/3 +CPP C3698 C0205770 C72.9 9390/0 +WT C3267 CL505178 C64.9 8960/3 683 +THYC C7569 C0205969 C37.9 8586/3 +TET C6450 C1266101 C37.9 8010/3 674 +HGNEC C96156 C3272607 C18.9 8246/3 +AODG C4326 C0334590 C72.9 9451/3 +PORO C27273 C1533161 +ILC C7950 C0279565 C50.9 8520/3 +HTAT C6846 C1266049 C73.9 8336/0 +LDD C8419 C0391826 C71.6 9493/0 +PBT C4952 C0750974 C71.9 8000/3 +ULM C3434 C0042133 C55.9 8890/0 +NMCHN C45716 C1707291 C76.0 8010/3 +UMLMS C40175 C1519861 C55.9 8896/3 +HEAD_NECK C12418 C0460004 C76.0 8000/3 608 +UDMN C36051 C1336860 C80.9 8000/3 +ACRM C4022 C0346037 C44.9 8744/3 +PHC C3326 C0031511 C74.9 8700/0 653 +DNT C9505 C1266177 C72.9 9413/0 +GRCT C3070 C0018206 C56.9 8620/1 +PENIS C12409 C0030851 650 +ACYC C2970 C0010606 C08.9 8200/3 +VPSCC C6982 C1336955 +CEEN C3769 C0206687 C53.9 8380/3 +CHS C2946 C0008479 C41.9 9220/3 +PTAD C3329 C0032000 C75.1 8272/0 +ESMM C5707 C1333460 C15.9 8746/3 +MNG C3230 C0025286 C70.9 9530/0 630 +LIPO C3194 C0023827 C49.9 8850/3 54056 +BEC C7010 C1333377 C71.9 9070/3 +CPT C3473 C0085138 C72.9 8000/3 +HPHSC C4043 C0280321 C13.9 8070/3 +LIHB C3728 C0206624 C22.0 8970/3 611 +DCIS C2924 C0007124 C50.9 8500/2 +TT C3877 C0238451 C62.9 9080/1 +MNGT C3229 C0025284 C70.9 9530/3 +ESCA C4025 C0279628 C15.9 8140/3 595 +CCBOV C40080 C0279676 +MCC C9231 C0007129 C44.9 8247/3 631 +ECD C53972 C0878675 C42.4 9750/3 594 +URCC C27892 C1336853 C64.9 8312/3 +DIPG C94764 C2986658 C72.9 9380/3 +HCL C7402 C0023443 C42.4 9940/3 607 +URCA C39842 C1511205 C67.7 8010/3 +USMT C40176 C1519863 C55.9 8897/1 +OMT C8112 C1334637 C56.9 9080/0 +ULMS C6340 C0280631 C55.9 8890/3 +MRT C3808 C0206743 C64.9 8963/3 +SEF C49027 C1710026 C49.9 8840/3 +AECA C6938 C1412016 C44.9 8401/3 +CENE C40214 C1516417 C53.9 8240/3 +LUCA C4038 C0280089 C34.9 8240/3 +TEOS C3902 C0259782 C41.9 9183/3 +MBEN C5407 C1334970 +OAST C4050 C0280793 C72.9 9382/3 +THYM C3411 C0040100 C37.9 8580/1 +DES C9182 C0079218 C49.9 8821/1 +PAASC C5721 C1335299 C25.9 8560/3 +SNSC C54287 C0334270 C30.0 8121/3 +THFO C8054 C0206682 C73.9 8331/3 +ALCL C3720 C0206180 C42.4 9714/3 560 +PECOMA C38150 C1300127 C49.9 8990/1 +BRCA C9245 C0853879 C50.9 8010/3 572 +UUS C8972 CL033042 C55.9 8930/3 +IHCH C35417 C0345905 C22.0 8160/3 +ODGC C4812 C0334558 C41.0 9270/3 +ACCC C3768 C0206685 C08.9 8550/3 +SKCN C3944 C1318558 C44.9 8761/1 +MMBC C40364 C1513365 C50.9 8575/3 +LYMPH C13252 C0024202 +LCLC C4450 C0345958 C34.9 8012/3 +STAS C5474 C1333761 C16.9 8560/3 +SCGBM C125890 C1272516 C72.9 9440/3 +COAD C4349 C0338106 C18.9 8140/3 585 +PGNT C92554 C2985174 C47.9 9509/1 +ESCC C4024 C0279626 C15.9 8070/3 595 +NLPHL C7258 C1334968 C42.4 9659/3 615 +FDCS C9281 C1260325 C42.4 9758/3 +EHAE C3800 C0206732 C49.9 9133/3 +EMBT C3264 C0027654 C72.9 9070/3 +READ C9383 C0149978 C20.9 8140/3 659 +CML C3172 C0023470 C42.4 9863/3 582 +CNC C3791 C0206719 C72.9 9506/1 +UCS C42700 C0280630 C55.9 8980/3 +IMMC C9131 C1334807 C50.9 8480/3 +MZL C4341 C1367654 C42.4 9699/3 626 +RAS C93125 C2985448 C49.9 8800/3 +SBWDNET C9461 C1332564 C17.9 8240/3 +MF C7052 C1266121 C49.9 8890/0 +MTSCC C39807 C1513719 C64.9 8032/3 +LIAS C4438 C0345907 C22.0 9124/3 +UCCC C6344 C1332912 C55.9 8310/3 +MP C3110 C0020217 +VYST C6379 C1336945 C57.9 9071/3 +HPCCNS C4660 C0349622 C72.9 9150/1 +SNUC C54294 C1710096 C30.0 8020/3 +HCCIHCH C3828 C0221287 C22.0 8180/3 +BREAST C12971 C0006141 C50.9 8000/3 44309 +EGC C9296 C1332166 C15.9 8140/3 +MYXO C6577 C0027149 C49.9 8840/0 +LIAD C3758 C0206669 C22.0 8170/0 +DF C6801 C0002991 C44.9 8832/0 +CCLC C4451 C1707407 C34.9 8310/3 +ETT C6900 C1266159 C55.9 9105/3 +IMT C6481 C0334121 C49.9 8825/1 +DDCHDM C48876 C1266174 C41.9 9372/3 +SBC C7724 C0238196 C17.9 8000/3 +LUAD C3512 C0152013 C34.9 8140/3 +MXOV C40090 C0279392 C56.9 8010/3 +SLCT C2880 C0003810 C56.9 8631/3 +INTS C53677 C1708550 C49.9 9137/3 +UAD C6167 C1336885 C68.0 8140/3 +PROSTATE C12410 C0033572 C61.9 8000/3 658 +UCU C6166 C0863015 C68.0 8120/3 +HCC C3099 C0019204 C22.0 8170/3 612 +PLMESO C9351 C1377913 C38.4 9050/3 632 +COADREAD C5105 C1319315 C18.9 8140/3 46096 +MPNST C3798 C0751690 C47.9 9540/3 +PTPR C92624 C2985219 C75.3 9395/3 +MPE C3697 C0205769 C72.9 9394/1 +CLNC C6905 C1370507 +DA C7889 C0278804 C17.0 8140/3 +PGNG C3308 C0030421 C49.9 8680/1 +HL C9357 C0019829 C42.4 9590/3 614 +MYCHS C4303 C0334551 C41.9 9231/3 +BONE C12366 C0262950 C41.9 8000/3 571 +IPMN C38342 C1518869 C25.9 8453/3 +MYEC C35700 C1335904 C08.9 8982/3 +BILIARY_TRACT C12678 C0005423 C24.9 8000/3 +CEGCC C40212 C1516407 C53.9 8015/3 +EMPD C3302 C0030186 C44.9 8542/3 +CHOS C4021 C0279603 C41.9 9181/3 +RLCLC C6876 C1265997 +CEMU C26712 C0007130 C53.9 8480/3 +PANET C27720 C1337011 C25.9 8150/1 647 +GINET C95404 C2987127 C26.9 8240/3 +SFT C7634 C1266119 C49.9 8815/0 +EPMT C6770 C1333407 C72.9 9391/3 +GCT C3474 C0085167 C72.9 9580/0 +HMBL C3801 C0206734 C72.9 9161/1 +PEOS C8970 C1377843 C41.9 9192/3 +SKAC C3775 C0206697 C44.9 8390/3 +HGSOS C53958 C1266165 C41.9 9194/3 +TLYM C6810 C0349644 C62.9 9590/3 +PNET C3716 C0206663 C72.9 9473/3 +DCS C9294 C1334030 +CCE C4714 C1384404 C72.9 9391/3 +PAST C4047 C0334583 C72.9 9421/1 +DFSP C4683 C0392784 C44.9 8832/3 +BCC C2921 C0007117 C44.9 8090/3 587 +THAP C3878 C0238461 C73.9 8021/3 +SKLMM C9151 C2739810 C44.9 8742/3 +TGCT C3401 C0039106 C49.9 9252/0 +PTCY C94524 C2986550 C75.1 9432/1 +UPECOMA C40180 C1519862 C55.9 8990/1 +SCRMS C6519 C1266134 C49.9 8912/3 +MCN C41247 C1518872 C25.9 8470/0 +GCCAP C3689 CL512511 C18.1 8243/3 +MACR C43585 C1707439 C18.9 8480/3 +AMBL C6904 C1266180 C72.9 9474/3 +IBC C4001 C0278601 C50.9 8530/3 +GTD C4699 C1135868 C55.9 9100/3 603 +CHGL C5592 C1322252 C71.9 9444/1 +RCC C9385 C0007134 C64.9 8312/3 660 +MNET C7172 C1332889 C72.9 8010/3 +VULVA C12408 C0042993 C57.9 8000/3 +SSRCC C5250 C1335965 C16.9 8490/3 +DASTR C7173 C0280785 C72.9 9400/3 +PRAD C2919 C0007112 C61.9 8140/3 +MSTAD C5248 C1334809 C16.9 8480/3 +OSOS C53953 C1704328 C41.9 9180/3 +PTES C27472 C1335563 C49.9 8804/3 +PPM C3904 C3163622 +VSC C7736 C0238518 C52.9 8070/3 680 +PMBL C9280 C1292754 C42.4 9679/3 657 +SPCC C45541 C1708784 C34.9 8801/3 +MTNN C3466 C0079772 C42.4 8000/3 46216 +HDCN C3106 C0019618 C42.4 8000/3 46099 +RDD C36075 C0019625 C42.4 9750/3 662 +MYELOID C12434 C0005767 +EMALT C3898 C0242647 C42.4 9699/3 +TLGL C4664 C1955861 C42.4 9831/3 621 +MBN C3457 CL448793 C42.4 8000/3 +CLLSLL C7540 C0855095 C42.4 9823/3 581 +PCM C3242 C0026764 C42.4 9732/3 633 +ADNOS C2852 C0001418 C80.9 8140/3 554 +AMLNPM1 C82431 C2826177 C42.4 9861/3 58170 +APLPMLRARA C9155 C0279625 C42.4 9866/3 553 +ATLL C3184 C0023493 C42.4 9827/3 556 +BLL C8936 C0023485 C42.4 9811/3 567 +BLLBCRABL1 C80331 C2698317 C42.4 9812/3 4770 +DLBCLNOS C8851 C0079744 C42.4 9680/3 589 +ENKL C4684 C0392788 598 +ET C3407 C0040028 C42.4 9962/3 596 +GBM C3058 C0017636 C72.9 9440/3 604 +GNOS C3059 C0017638 C72.9 9380/3 623 +HGGNOS C72.9 9380/3 46077 +LATL C42.4 9970/1 46091 +LBGN C42.4 8000/0 46091 +LCS C6921 C1260327 620 +LNM C7065 C0598798 C42.4 8000/3 46214 +MIDDA C2868 C0002726 622 +SS C3366 C0036920 C42.4 9701/3 586 +NETNOS C3809 C0206754 C80.9 8240/3 636 +PMF C2862 C0001815 C42.4 9961/3 +PVMF C41233 C3805232 C42.4 9950/3 +ETMF C126806 C3805233 C42.4 Unknown +PTCL C4340 C2853959 C42.4 9702/3 652 +PTLD C4727 C0432487 C42.4 9971/1 656 +PV C3336 C0032463 C42.4 9950/3 655 +SARCNOS C9118 C1261473 C49.9 8800/3 46222 +SCCNOS C2929 C0007137 C80.9 8070/3 663 +TLL C8694 C1301359 C42.4 9837/3 670 +TPLL C4752 C2363142 C42.4 9834/3 50421 +IPN C6881 C1879344 +IUP C6192 C1334282 +URMM C68.0 8746/3 +UPA C39858 C1384678 C67.9 8120/1 +ADMA C7644 C0334556 C41.9 9310/0 +LAMN C42598 C1708747 +TAC C7041 C1112503 C18.9 8211/0 +BNNOS C2910 C1458155 C50.9 8000/3 +JSCB C4189 C0334371 +DIFG C129325 CL512143 C72.9 9380/3 +ENCG C72.9 8000/3 +BGCT C5795 CL448320 C71.9 9064/3 +PCNSMT C72.9 8000/3 +CELI C128047 CL509701 +CERMS C128048 CL509707 +MCCE C53.9 8000/3 +EPDCA C95612 C2987256 C15.9 8010/3 +SMN C3751 C0206658 C16.9 8897/1 +LGT C12346 C0022907 +OHNCA C35850 C3887461 C76.0 8010/3 +HNMUCM C133187 CL520085 C76.0 8746/3 +PTH C4906 C0687150 +SBL C35837 C1335911 +CCSK C4264 C0334488 +RNET C157743 CL937404 C64.9 8000/3 +MRTL C96847 C3273076 +UESL C27096 C0855073 +LAIS C136486 CL523789 C34.9 8140/2 +MATPL C42.4 9876/3 +MBGN C42.4 8000/0 +MNM C9290 C2939461 C42.4 8000/1 +EGCT C3918 C3544268 +MIXED C80.9 8000/3 +OOVC C4908 C0677886 C56.9 8010/3 +GN C3049 C0017075 +PSEC C48.2 8461/3 +BCCP C39902 C1514507 +AN C3694 C0205748 C44.9 8727/0 +EMPSGC C167364 CL978748 C44.9 8481/3 +AFH C6494 C1266127 +IFS C4244 C0334459 +LM C3157 C0023267 +MGST C4221 C1266111 +IMS C3742 C0206648 +RCSNOS C49.9 8803/3 +TMESO C62.9 9050/3 +TNET C37.9 8240/3 +OAT C6042 C1336750 C73.9 8000/0 +OUTT C55.9 8000/1 +VGCT C128294 CL509570 +VPDC C52.9 8010/3 +PAMPCA C24.1 8163/3 +EHCH C24.9 8160/3 +GBASC C7356 C1333741 C23.9 8560/3 +GBAD C9166 C0279651 C23.9 8140/3 +SCGBC C6763 C1333759 +CCHDM C41.9 9370/3 +CTAAP C18.1 8140/3 +CAIS C18.9 8140/2 +BRCANOS C50.9 8000/3 +BRCNOS C9245 C0853879 C50.9 8010/3 +CSNOS C50.9 8980/3 +EMBC C40364 C1513365 C50.9 8575/3 +APXA C129327 CL512339 C72.9 9424/3 +LGGNOS C132067 C1997217 C72.9 9380/3 +BYST C71.9 9071/3 +SFTCNS C129526 CL512688 C72.9 8815/0 +HGNET C72.9 8010/3 +LGNET C72.9 8010/3 +MELC C9498 C0334431 +APTAD C132296 CL520582 C75.1 8140/1 +ACPG C4726 C0431129 C72.9 9351/1 +PCGP C4725 C0431128 C72.9 9352/1 +CESE C53.9 8441/3 +GRC C16.9 8140/3 +ASCT C02.9 8560/3 +HNNE C76.0 8246/3 +HNSCUP C76.0 8070/3 +PTHC C4906 C0687150 C75.0 8010/3 +HNMASC C123384 CL512709 C08.9 8502/3 +OSACA C08.9 8010/3 +NCCRCC C64.9 8010/3 +CMPT C34.9 8050/3 +NUTCL C146706 CL544717 +NSCLCPD C34.9 8046/3 +SGTTL C34.9 8200/3 +ALAL C7464 C1301357 C42.4 9801/3 +MCD C84269 C0024899 +MDS/MPN C27262 C1301355 C42.4 9975/3 +MNGLP C130038 CL514062 +MLNER C84270 C2827356 +ACN C3768 C0206685 C80.9 8550/3 +CUPNOS C80.9 8000/3 +NECNOS C3773 C0206695 C80.9 8246/3 +PDC C80.9 8010/3 +SCUP C3915 C0262584 C80.9 8042/3 +HGONEC C5238 C1335174 C56.9 8246/3 +HGSFT C57.0 8461/3 +OCNOS C4515 C0346181 +OSMAD C4508 C0346166 +SBMOV C122585 C4055371 C56.9 8460/3 +FT C3405 C0039747 C56.9 8600/0 +SCT C4215 C0334412 C56.9 8670/0 +ITPN C95506 C2987189 +PLBMESO C45665 C1709570 C38.4 9053/3 +PLEMESO C45662 C1709574 C38.4 9052/3 +PLSMESO C45663 C1709578 C38.4 9050/3 +MUP C154473 CL555395 C80.9 8720/3 +SPZM C165497 C3495721 C44.9 8772/3 +UPDC C55.9 8010/3 +UDDC C55.9 8010/3 +UMNC C55.9 9110/3 +UMEC C54.1 8010/3 +UNEC C126771 CL508215 C55.9 8246/3 +OUSARC C55.9 8800/3 +VOEC C3752 C0206659 C57.9 9070/3 +VPE C8109 +GCEMU C53.9 8480/3 +SPDAC C16.9 8010/3 +SCCRCC C27893 C1266043 C64.9 8318/3 +CCPRC C121955 C4054076 C64.9 8050/3 +FHRCC C164156 CL977349 C64.9 8312/3 +BLLRGA C80328 C2698313 C42.4 9811/3 +BLLNOS C80326 C2698310 C42.4 9811/3 +CHLPTLD C7243 C1334037 +FHPTLD C139028 CL526535 +IMPTLD C7236 C1334174 +MPTLD C7233 C1334798 +PHPTLD C7235 C1335428 +PPTLD C7183 C1301361 +ETPLL C130043 CL514044 C42.4 9837/3 +NKCLL C82217 C2826059 +AUL C9298 C1378511 C42.4 9801/3 +MPALBCRABL1 C82192 C2826037 C42.4 9806/3 +MPALKMT2A C82203 C2826048 C42.4 9807/3 +MPALBNOS C82212 C3472616 C42.4 9808/3 +MPALTNOS C82213 C2826055 C42.4 9809/3 +AMLMRC C7600 C2825139 C42.4 9895/3 +AMLRGA C7175 C1275661 C42.4 9861/3 +AMLNOS C27753 CL054841 C42.4 9861/3 +MPRDS C82338 C2826104 +MS C3520 CL414320 C42.4 9930/3 +TMN C27912 C1292776 C42.4 9920/3 +JXG C81772 C2825742 C42.4 8000/3 +FRCT C81758 C2825739 +HS C27349 C0334663 +IDCT C81767 C2825741 +CMCD C7137 C1136033 +MCSL C9348 C0036221 +MDSEB C7506 C0002894 C42.4 9983/3 +MDSID5Q C6867 C1292779 C42.4 9986/3 +MDSMD C8574 C0796466 C42.4 9985/3 +MDSRS C82616 C2826330 C42.4 9982/3 +MDSSLD C82591 C2826318 C42.4 9989/3 +MDSU C8648 C0854809 C42.4 9989/3 +RCYC C82596 C2826323 +ACML C3519 C0349640 C42.4 9876/3 +JMML C9233 C0349639 C42.4 9946/3 +MDSMPNRST C82616 C2826330 C42.4 9982/3 +MDSMPNU C27780 C1328061 C42.4 9975/3 +MLNFGFR1 C84277 C2827362 +MLNPCM1JAK2 C129853 CL512939 +MLNPDGFRA C84275 C2827360 C42.4 9965/3 +MLNPDGFRB C84276 C3472621 +CELNOS C4563 C0346421 C42.4 9964/3 +CNL C3179 C0023481 C42.4 9963/3 +MPNU C27350 C1333046 C42.4 8000/0 +BTBEOV C4746 C0474834 +BTBOV C9459 C0334494 +BTMOV C4270 C0334495 +HGESS C126998 CL508195 C54.1 8930/3 +ABC C36081 C1333296 +GCB C38335 C1517532 C42.4 9680/3 +DFL C138185 CL525720 +ISFN C138181 CL525716 +ISMCL C138191 CL525722 +MGUSIGA C565 C0020835 +MGUSIGG C568 C0020852 +MGUSIGM C569 C0020861 +MIDDO C7151 CL357425 +HCL-V C7401 C0349633 +SDRPL C80309 C2699508 +ALCLALKN C37194 C1332078 +ALCLALKP C37193 C1332079 +BIALCL C139012 CL526545 C42.4 9702/3 +LYP C3721 C0206182 +PCALCL C6860 C1301362 +BLLHYPER C80335 C2698311 C42.4 9815/3 +BLLHYPO C80338 C2698312 C42.4 9816/3 +BLLIAMP21 C130039 CL513772 +BLLETV6RUNX1 C80334 C2698314 +BLLTCF3PBX1 C80341 C2698315 C42.4 9818/3 +BLLIL3IGH C80340 C2698316 +BLLKMT2A C80332 C2698309 C42.4 9813/3 +BLLBCRABL1L C129788 CL512918 +LDCHL C9283 C0152267 +LRCHL C6913 C1266194 +MCCHL C3517 C0152266 +NSCHL C3518 C0152268 +ALKLBCL C7225 C1333294 +AHCD C3132 C0021071 +BCLU C37869 C1333878 +BPLL C4753 C0475801 +BLL11Q C131911 CL520452 +DLBCLCI C80289 C2699776 +EBVDLBCLNOS C80281 C2700007 C42.4 9680/3 +EBVMCU C131906 CL519556 +EP C4002 C0278619 +GHCD C3083 C0018854 +HHV8DLBCL C138320 CL525847 +HGBCL C80291 C2698294 +HGBCLMYCBCL2 C131913 CL524351 C42.4 9680/3 +IVBCL C4342 C0334660 +LBLIRF4 C133494 CL521195 +LYG C7930 C0024307 +LPL C3212 C0334633 +MCBCL C80310 C2698259 +MGUS C3996 C0026470 +MIDD C7151 CL357425 +MHCD C3892 C0242310 +PTFL C80297 C2698750 +PLBL C7224 C3472614 +PCLBCLLT C45194 C1709656 +PCFCL C7217 C1333171 +SPB C7812 C0272256 +SBLU C80308 C2699507 +THRLBCL C9496 C1321547 C42.4 9688/3 +ANKL C8647 C1292777 +CLPDNK C39591 C1512709 +EATL C4737 C0456889 +FTCL C80375 C2700204 +HSTCL C8459 C0522627 +HVLL C45327 C1708397 +ITLPDGI C139021 CL526526 +MEITL C96058 C3272525 +NPTLTFH C139011 CL526546 +PCATCL C139023 CL526531 +PCLPD C7195 C1371159 +PCSMTPLD C45366 CL524855 +PCAECTCL C45339 C1709653 C42.4 9709/3 +PCGDTCL C45340 C1707547 +SPTCL C6918 C0522624 +SEBVTLC C80374 C2699747 +AMLRBM15MKL1 C82427 C2826173 +AMLBCRABL1 C129785 CL512855 C42.4 9861/3 +AMLCEBPA C129782 CL512856 C42.4 9861/3 +AMLRUNX1 C129786 CL512858 +AMLRARA C36055 C2825128 +AMLCBFBMYH11 C9287 C0522630 C42.4 9871/3 +AMLGATA2MECOM C82426 C2826172 C42.4 9869/3 +AMLDEKNUP214 C82423 C2826169 C42.4 9865/3 +AMLRUNX1RUNX1T1 C9288 C1292774 C42.4 9896/3 +AMLMLLT3KMT2A C156719 CL935859 C42.4 9897/3 +AM C7961 C0279624 C42.4 9874/3 +AMLMD C8460 C0522631 C42.4 9872/3 +AWM C9380 C0279623 C42.4 9873/3 +ABL C3164 C0023437 +AMKL C3170 C0023462 C42.4 9910/3 +AMML C7463 C0023479 C42.4 9867/3 +APMF C4344 C0334674 C42.4 9931/3 +PERL C7467 CL028054 C42.4 9840/3 +MLADS C43223 C2825149 +TAM C82339 C1834582 +TAML C8252 C1336735 C42.4 9920/3 +TMDS C27722 C1292780 C42.4 9987/3 +ASM C9285 C1112486 +ISM C9286 C0272203 C42.4 9741/1 +SMMCL C3169 C0023461 +SSM C115460 CL388369 +SMAHN C9284 C1301365 C42.4 9741/3 +MDSEB1 C7167 CL514065 C42.4 9983/3 +MDSEB2 C7168 CL514066 C42.4 9983/3 +MDSRSMD C27726 C1301356 C42.4 9982/3 +MDSRSSLD C130037 CL514168 C42.4 9982/3 +CMML0 C130035 CL513815 C42.4 9945/3 +CMML1 C36061 C1333043 C42.4 9945/3 +CMML2 C36062 C1333044 C42.4 9945/3 +CMLBCRABL1 C9128 C1292771 C42.4 9876/3 +PMFPES C41237 C1516553 C42.4 9961/3 +PMFOFS C41238 C1516552 C42.4 9961/3 diff --git a/scripts/ontology_to_ontology_mapping_tool/ontology_to_ontology_mapping_tool.py b/scripts/ontology_to_ontology_mapping_tool/ontology_to_ontology_mapping_tool.py new file mode 100644 index 00000000..6badca2c --- /dev/null +++ b/scripts/ontology_to_ontology_mapping_tool/ontology_to_ontology_mapping_tool.py @@ -0,0 +1,163 @@ +# Copyright (c) 2020 Memorial Sloan-Kettering Cancer Center. +# +# This library is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF +# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. The software and +# documentation provided hereunder is on an "as is" basis, and +# Memorial Sloan-Kettering Cancer Center +# has no obligations to provide maintenance, support, +# updates, enhancements or modifications. In no event shall +# Memorial Sloan-Kettering Cancer Center +# be liable to any party for direct, indirect, special, +# incidental or consequential damages, including lost profits, arising +# out of the use of this software and its documentation, even if +# Memorial Sloan-Kettering Cancer Center +# has been advised of the possibility of such damage. + +import sys +import os +import io +import argparse +import pandas as pd +import requests + +ONTOLOGY_MAPPINGS_FILE_URL = "https://raw.githubusercontent.com/cBioPortal/oncotree/master/scripts/ontology_to_ontology_mapping_tool/ontology_mappings.txt" +VALID_ONCOTREE_CODES_URL = "https://raw.githubusercontent.com/cBioPortal/oncotree/master/resources/resource_uri_to_oncocode_mapping.txt" +ACCEPTED_ONTOLOGIES = ['ONCOTREE_CODE', 'UMLS_CODE', 'NCIT_CODE', 'ICDO_TOPOGRAPHY_CODE', 'HEMEONC_CODE', 'ICDO_MORPHOLOGY_CODE'] +NULL_VALUES = ['', 'NA'] + +def add_comments_column_and_log_data(mapped_data, target_file, source_code, target_code, source_file): + comments = [] + completely_resolved_codes = {} + ambiguous_codes = {} + unrecognized_codes = [] + many_to_one_codes = {} + mapped_codes_count = 0 + unmapped_codes_count = 0 + + columns_to_groupby = mapped_data.columns[:len(mapped_data.columns)-1] + grouped_data = mapped_data.groupby(list(columns_to_groupby), sort=False)[target_code].unique().apply(', '.join).reset_index() + + oncotree_codes_list = pd.read_csv(io.StringIO(requests.get(VALID_ONCOTREE_CODES_URL).content.decode('utf-8')), sep='\t', header=None, keep_default_na=False) + valid_oncotree_codes = oncotree_codes_list.loc[oncotree_codes_list[1] == "hasCode"][2].str.upper().tolist() + + for sc, tc in zip(grouped_data[source_code], grouped_data[target_code]): + if tc == "": #one_to_none mapping + if source_code == "ONCOTREE_CODE" and sc not in valid_oncotree_codes: + comments.append("Invalid code") + else: + comments.append((lambda sc: 'No mapping found' if sc not in NULL_VALUES else '')(sc)) + unrecognized_codes.append(sc) + if sc not in NULL_VALUES: + unmapped_codes_count += 1 + elif len(tc.split(', ')) > 1: #one_to_many mapping + comments.append('Maps to multiple codes') + ambiguous_codes[sc] = tc + mapped_codes_count += 1 + else: #one_to_one mapping + comments.append('') + completely_resolved_codes[sc] = tc + mapped_codes_count += 1 + + #Also print many_to_one mapping to Log file (Multiple source codes map to one target code) + many_to_one = mapped_data.groupby(target_code, sort=False)[source_code].unique().apply(', '.join).reset_index() + for tc, sc in zip(many_to_one[target_code], many_to_one[source_code]): + if tc != '' and len(sc.split(', ')) > 1: + many_to_one_codes[tc] = sc + + grouped_data['COMMENTS'] = comments + grouped_data.to_csv(target_file, sep='\t', index=False) + + log_file = open(os.path.splitext(target_file)[0] + "_summary.html", 'w') + log_file.write("\n\n\nOntology Mapping Summary\n\n\n\n") + log_file.write("

Ontology Mapping Summary

\n") + log_file.write("Source Ontology: %s
" % (source_code)) + log_file.write("Target Ontology: %s

" % (target_code)) + log_file.write("Mapped %s to %s

" % (source_code, target_code)) + log_file.write("
Total Mapped Entries %s
Total Unmapped Entries %s
" % (mapped_codes_count, unmapped_codes_count)) + + if completely_resolved_codes: + log_file.write("

The following source codes mapped to one target code:

\n") + for code in completely_resolved_codes: + log_file.write("

Source Code: %s
\n" % (code)) + log_file.write("Target Code: %s
\n" % completely_resolved_codes[code]) + if ambiguous_codes: + log_file.write("

The following source codes mapped to multiple target codes:

\n") + for code in ambiguous_codes: + log_file.write("

Source Code: %s
\n" % (code)) + log_file.write("Target Codes: %s
\n" % ambiguous_codes[code]) + if unrecognized_codes: + log_file.write("

The following source codes have no mapping available:

\n") + for code in unrecognized_codes: + log_file.write("

Source Code: %s
\n" % ("<blank>" if len(code) == 0 else code)) + log_file.write("Mapping not available for the code\n") + if many_to_one_codes: + log_file.write("

The following target codes mapped to multiple source codes:

\n") + for code in many_to_one_codes: + log_file.write("

Target Code: %s
\n" % (code)) + log_file.write("Source Codes: %s
\n" % many_to_one_codes[code]) + + log_file.close() + +#Check if any of the ontology code headers are present in the clinical file +def validate_arguments(source_file, source_code, target_code): + #1. Check if the source_code input by the user is valid. + if source_code not in ACCEPTED_ONTOLOGIES: + print("Invalid source code: \'%s\'. \nThe list of acceptable codes are:" % (source_code)) + for code in ACCEPTED_ONTOLOGIES: + print(code) + sys.exit(1) + + #2. Check if the target_code input by the user is valid. + if target_code not in ACCEPTED_ONTOLOGIES: + print("Invalid target code: \'%s\'. \nThe list of acceptable codes are:" % (target_code)) + for code in ACCEPTED_ONTOLOGIES: + print(code) + sys.exit(1) + + #3. Check if source_code and target_code are not the same + if source_code == target_code: + print("The source and target ontology columns to be mapped are the same.\nSource Ontology: %s\nTarget Ontology: %s" % (source_code, target_code)) + print("Please specify different ontologies to map on.\n") + sys.exit(1) + + #4. CHeck if the user input file has the source code column. + source_file.columns = map(str.upper, source_file.columns) + if source_code not in source_file.columns: + print("\nThe input file does not contain the requested source ontology column or the headers do not match to accepted values. Please check. \n\nThe list of acceptable ontology headers are:") + for code in ACCEPTED_ONTOLOGIES: + print(code) + print('\n') + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--source-file', required = True, help = 'This is the source file path. The source file must contain one of the ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE in the file header and it must contain codes corresponding to the Ontology System.', type = str) + parser.add_argument('-o', '--target-file', required = True, help = 'This is the path to the target file that will be generated. It will contain the ontology mappings of source code in to .', type = str) + parser.add_argument('-s', '--source-code', required = True, help = "This is the source ontology code in . It must be one of the ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE.", type = str) + parser.add_argument('-t', '--target-code', required = True, help = "This is the target ontology code that the script will attempt to map the source ontology code to. It must be one of the ONCOTREE_CODE, NCIT_CODE, UMLS_CODE, ICDO_TOPOGRAPHY_CODE, ICDO_MORPHOLOGY_CODE or HEMEONC_CODE.", type = str) + args = parser.parse_args() + + target_file = args.target_file + source_code = args.source_code.upper() + target_code = args.target_code.upper() + + source_file = pd.read_csv(args.source_file, comment='#', sep='\t', header=0, keep_default_na=False).applymap(str) #HEMEONC_CODES are numbers. + validate_arguments(source_file, source_code, target_code) + + oncotree_code_source_data = requests.get(ONTOLOGY_MAPPINGS_FILE_URL).content + mappings_file = pd.read_csv(io.StringIO(oncotree_code_source_data.decode('utf-8')), sep='\t', header=0, keep_default_na=False).applymap(str) + + #For case insensitive merge. + source_file[source_code] = source_file[source_code].str.upper() + mappings_file[source_code] = mappings_file[source_code].str.upper() + + mapped_data = pd.merge(source_file, mappings_file[[source_code, target_code]], on=source_code, suffixes= ('_source_file', ''), sort=False, how='left').fillna('') + print("Mapped the %s to %s.." % (source_code, target_code)) + + add_comments_column_and_log_data(mapped_data, target_file, source_code, target_code, source_file) + print("\nThe ontology mappings are written to: %s" % (target_file)) + print("The mapping summary is written to : %s" % (os.path.splitext(target_file)[0] + "_summary.html")) + +if __name__ == '__main__': + main() diff --git a/scripts/test/__init__.py b/scripts/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/test/test_oncotree_to_oncotree.py b/scripts/test/test_oncotree_to_oncotree.py new file mode 100644 index 00000000..4e52012a --- /dev/null +++ b/scripts/test/test_oncotree_to_oncotree.py @@ -0,0 +1,265 @@ +# run all unit tests with: +# scripts> python -m unittest discover +# +# Author: Manda Wilson + +import unittest + +from oncotree_to_oncotree import * + +class TestCrossVersionOncotreeTranslator(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.original_version = cls.get_original_version() + for code in (cls.original_version): + GLOBAL_LOG_MAP[code] = { + NEIGHBORS_FIELD : [], + CHOICES_FIELD : [], + CLOSEST_COMMON_PARENT_FIELD : "", + IS_LOGGED_FLAG : False + } + cls.latest_version = cls.get_latest_version() + for code in (cls.latest_version): + GLOBAL_LOG_MAP[code] = { + NEIGHBORS_FIELD : [], + CHOICES_FIELD : [], + CLOSEST_COMMON_PARENT_FIELD : "", + IS_LOGGED_FLAG : False + } + + # --------------------------------------------------------------------------------- + # Original Code: SEZS (Child: FAKE_OLD_SS_CHILD) + # New Code: SS Child: FAKE_OLD_SS_CHILD, FAKE_NEW_SS_CHILD) + # Code is mapped through history. One new child is introduced in the future version (SS) + # Tests simple direct mapping with and without introduction of child (forward, backward) + + # get possible codes (jumping forward through history) + def test_get_possible_oncotree_code_forwards_history(self): + self.run_get_possible_oncotree_code_test("SEZS", set(["SS"]), False) + + # get possible codes (jumping backwards through history) + def test_get_possible_oncotree_code_backwards_history(self): + self.run_get_possible_oncotree_code_test("SS", set(["SEZS"]), True) + + # get resolved string - single direct mapping, new children available + def test_resolve_single_future_possible_target_oncotree_code_with_new_child(self): + self.run_resolve_oncotree_codes_test("SEZS", set(["SS"]), False, format_oncotree_code_options("SEZS", "{SS}", 1), False) + + # get resolved string - single direct mapping, no new children + def test_resolve_single_past_possible_target_oncotree_code(self): + self.run_resolve_oncotree_codes_test("SS", set(["SEZS"]), True, "SEZS", True) + + def test_get_number_of_new_children_backwards(self): + self.run_get_number_of_new_children_test("SS", set(["SEZS"]), True, 0) + + def test_get_number_of_new_children_fowards(self): + self.run_get_number_of_new_children_test("SEZS", set(["SS"]), False, 1) + # --------------------------------------------------------------------------------- + # Original Code: ALL (Children: BALL, TALL, DALL) + # New Code: BLL, TLL, DLL (No children) + # Original code's children are precursors/history to new codes: precursor (BALL -> BLL, TALL -> TLL), history (DALL -> DLL) + # ALL is in BLL, TLL revocations NOT DLL + + # ALL forward should map to "BLL" and "TLL" (not "DLL"), reverse mapping should not return "ALL" + def test_get_possible_oncotree_code_forwards_two_revocations(self): + self.run_get_possible_oncotree_code_test("ALL", set(["BLL", "TLL"]), False) + + def test_get_possible_oncotree_code_forwards_revocation_and_precursor(self): + self.run_get_possible_oncotree_code_test("BALL", set(["BLL"]), False) + + def test_get_possible_oncotree_code_forwards_revocation_and_history(self): + self.run_get_possible_oncotree_code_test("DALL", set(["DLL"]), False) + + # testing revocation - "BLL" should not include "ALL" as option + def test_get_possible_oncotree_code_backwards_revocation_and_precursor(self): + self.run_get_possible_oncotree_code_test("BLL", set(["BALL"]), True) + + def test_get_possible_oncotree_code_backwards_revocation_and_history(self): + self.run_get_possible_oncotree_code_test("DLL", set(["DALL"]), True) + + def test_resolve_multiple_future_possible_target_oncotree_codes_no_new_children(self): + self.run_resolve_oncotree_codes_test("ALL", set(["BLL", "TLL"]), False, format_oncotree_code_options("ALL", "{BLL,TLL}", 0), False) + # --------------------------------------------------------------------------------- + # Original Code: GMUCM + # New Code: URMM + # Original code was revoked and replaced with URMM + # URMM is effectively a new node and cannot be mapped directly back + + def test_get_possible_oncotree_code_backwards_revocation(self): + self.run_get_possible_oncotree_code_test("URMM", set([]), True) + + def test_get_possible_oncotree_code_forwards_revocations(self): + self.run_get_possible_oncotree_code_test("GMUCM", set(["URMM"]), False) + + # going backwards - no choices, search neighborhood - skip BLADDER because not mappable - map non-immediate neighbor TISSUE + def test_resolve_no_past_possible_target_oncotree_codes(self): + self.run_resolve_oncotree_codes_test("URMM", set(), True, format_oncotree_code_options("URMM", "Neighborhood: TISSUE", 0), False) + # --------------------------------------------------------------------------------- + # Original Code: CLL, SLL + # New Code: CLLSLL + # Original code was combined into CLLSLL (both are precursors) + # CLLSLL can be mapped back to either + + def test_get_possible_oncotree_code_backwards_merged_precusors(self): + self.run_get_possible_oncotree_code_test("CLLSLL", set(["CLL", "SLL"]), True) + + def test_get_possible_oncotree_code_forwards_merged_precusors(self): + self.run_get_possible_oncotree_code_test("CLL", set(["CLLSLL"]), False) + + def test_resolve_multiple_past_possible_target_oncotree_codes(self): + self.run_resolve_oncotree_codes_test("CLLSLL", set(["CLL", "SLL"]), True, format_oncotree_code_options("CLLSLL", "{CLL,SLL}", 0), False) + # --------------------------------------------------------------------------------- + # Original Code: PTCL, PTCLNOS + # New Code: PTCL (originally PTCLNOS) + # PTCL (the meaning) was revoked - PTCLNOS (the meaning) stayed but was renamed to PTCL + + def test_get_possible_oncotree_code_forwards_revoked_and_renamed(self): + self.run_get_possible_oncotree_code_test("PTCL", set(["PTCL"]), False) + + def test_get_possible_oncotree_code_forwards_revoked_and_renamed(self): + self.run_get_possible_oncotree_code_test("PTCLNOS", set(["PTCL"]), False) + + # in this test, we see that the revoked parent node (PTCL) is not chosen as a valid backwards mapping. Only the history (PTCLNOS) of the test node (PTCL) is considered valid + def test_get_possible_oncotree_code_backwards_revoked_and_renamed(self): + self.run_get_possible_oncotree_code_test("PTCL", set(["PTCLNOS"]), True) + # --------------------------------------------------------------------------------- + # Original Code: TNKL (parent), CTCL, TNKL_CHILD, PTCL (children) + # New Code: TNKL gone, CTCL renmaed to MYCF, PTCL revoked by PTCLNOS (renamed PTCL), TNKL_CHILD renamed to TNKL + # TNKL can't be mapped forward - possible set should be 0 + + # Tests for TNKL + def test_get_no_possible_oncotree_code_forwards_multiple_children(self): + self.run_get_possible_oncotree_code_test("TNKL", set(), False) + + def test_resolve_multiple_past_possible_target_oncotree_codes(self): + self.run_resolve_oncotree_codes_test("TNKL", set(), False, format_oncotree_code_options("TNKL", "Neighborhood: MYCF,PTCL,TISSUE,TNKL_CHILD2", 0), False) + + def test_get_possible_oncotree_code_forwards_precursors(self): + self.run_get_possible_oncotree_code_test("CTCL", set(["MYCF"]), False) + + def test_get_possible_oncotree_code_backwards_precursors(self): + self.run_get_possible_oncotree_code_test("MYCF", set(["CTCL"]), True) + # --------------------------------------------------------------------------------- + # Tests for getting valid neighboring OncoTree codes + + # TNKL_NEW_CHILD parent MTNN is not in target (original) version + # TNKL_GRANDCHILD is in both + def test_get_neighbor_invalid_parent_valid_children(self): + self.run_get_neighboring_target_oncotree_codes_test(["TNKL_NEW_CHILD"], True, set(["TNKL_GRANDCHILD"])) + + # TNKL_NEW_CHILD2 parent MTNN is not in target (original) version + # skip to grandparent "TISSUE" + # no children + def test_get_neighbor_invalid_parent_invalid_children(self): + self.run_get_neighboring_target_oncotree_codes_test(["TNKL_NEW_CHILD2"], True, set(["TISSUE"])) + + # FAKE_NEW_SS_CHILD has no children + # parent ("SS") maps back to "SEZS" in target version + def test_get_neighbor_valid_parent_invalid_children(self): + self.run_get_neighboring_target_oncotree_codes_test(["FAKE_NEW_SS_CHILD"], True, set(["SEZS"])) + + # BLADDER has children CLLSLL, URMM + # URMM revoked GMUCM (GMUCM is no longer valid) - don't map back + # CLLSLL maps back to CLL or SLL + def test_get_neighbor_partially_valid_children_valid_parent(self): + self.run_get_neighboring_target_oncotree_codes_test(["BLADDER"], True, set(["CLL", "SLL", "TISSUE"])) + # --------------------------------------------------------------------------------- + # Tests for getting closest common parent for a set of nodes + + def test_get_closest_common_parent_for_distant_nodes(self): + self.run_get_closest_common_parent_test(["TNKL_GRANDCHILD", "CLL", "PTCLNOS", "MEL"], self.original_version, "TISSUE") + + def test_get_closest_common_parent_for_parent_child_nodes(self): + self.run_get_closest_common_parent_test(["SEZS", "CTCL", "FAKE_OLD_SS_CHILD"], self.original_version, "CTCL") + + def test_get_closest_common_parent_for_related_child_nodes(self): + self.run_get_closest_common_parent_test(["TALL", "BALL", "DALL"], self.original_version, "ALL") + # --------------------------------------------------------------------------------- + # Test functions + def run_get_possible_oncotree_code_test(self, test_oncotree_code, expected_possible_oncotree_codes, is_backwards_mapping): + source_version = self.original_version if not is_backwards_mapping else self.latest_version + target_version = self.latest_version if not is_backwards_mapping else self.original_version + actual_output = get_possible_target_oncotree_codes(source_version[test_oncotree_code], target_version, is_backwards_mapping) + self.assertEqual(expected_possible_oncotree_codes, actual_output) + + def run_resolve_oncotree_codes_test(self, source_oncotree_code, possible_target_oncotree_codes, is_backwards_mapping, expected_oncotree_code_option, expected_is_easily_resolved): + source_version = self.original_version if not is_backwards_mapping else self.latest_version + target_version = self.latest_version if not is_backwards_mapping else self.original_version + actual_oncotree_code_option, actual_is_easily_resolved = resolve_possible_target_oncotree_codes(source_oncotree_code, possible_target_oncotree_codes, source_version, target_version, is_backwards_mapping) + self.assertEqual(expected_is_easily_resolved, actual_is_easily_resolved) + # since neighborhood returns a set with no order, check contents instead + # XYZ -> Neighborhood: X,Y,Z + # split ['XYZ -> Neighborhood', 'X,Y,Z'] + # split ['X,Y,Z'] to ['X', 'Y', 'Z'] + if "Neighborhood" in expected_oncotree_code_option: + expected_oncotree_codes = expected_oncotree_code_option.split("Neighborhood: ")[1].split(",") + actual_oncotree_codes = actual_oncotree_code_option.split("Neighborhood: ")[1].split(",") + self.assertEqual(set(expected_oncotree_codes), set(actual_oncotree_codes)) + else: + self.assertEqual(expected_oncotree_code_option, actual_oncotree_code_option) + + def run_get_neighboring_target_oncotree_codes_test(self, source_oncotree_codes, is_backwards_mapping, expected_neighbors): + source_version = self.original_version if not is_backwards_mapping else self.latest_version + target_version = self.latest_version if not is_backwards_mapping else self.original_version + actual_neighbors = get_neighboring_target_oncotree_codes(source_oncotree_codes, source_version, target_version, True, is_backwards_mapping) + self.assertEqual(expected_neighbors, actual_neighbors) + + def run_get_number_of_new_children_test(self, source_oncotree_code, possible_target_oncotree_codes, is_backwards_mapping, expected_number_of_new_children): + source_version = self.original_version if not is_backwards_mapping else self.latest_version + target_version = self.latest_version if not is_backwards_mapping else self.original_version + actual_number_of_new_children = get_number_of_new_children(source_oncotree_code, possible_target_oncotree_codes, source_version, target_version) + self.assertEqual(expected_number_of_new_children, actual_number_of_new_children) + + def run_get_closest_common_parent_test(self, possible_target_oncotree_codes, target_oncotree, expected_closest_common_parent): + actual_closest_common_parent = get_closest_common_parent(possible_target_oncotree_codes, target_oncotree) + self.assertEqual(expected_closest_common_parent, actual_closest_common_parent) + + @classmethod + def get_original_version(cls): + return { + "ALL" : {"code": "ALL", "children": ["TALL", "BALL", "DALL"], "parent": "LEUK", "revocations": [], "precursors": [], "history": []}, + "BALL" : {"code": "BALL", "children": [], "parent": "ALL", "revocations": [], "precursors": [], "history": []}, + "CLL" : {"code": "CLL", "children": [], "parent": "LEUK", "revocations": [], "precursors": [], "history": []}, + "CTCL" : {"code": "CTCL", "children": ["SEZS"], "parent": "TNKL", "revocations": [], "precursors": [], "history": []}, + "DALL" : {"code": "DALL", "children": [], "parent": "ALL", "revocations": [], "precursors": [], "history": []}, + "FAKE_OLD_SS_CHILD" : {"code": "FAKE_OLD_SS_CHILD", "children": [], "parent": "SEZS", "revocations": [], "precursors": [], "history": [""]}, + "GMUCM" : {"code": "GMUCM", "children": [], "parent": "MEL", "revocations": [], "precursors": [], "history": []}, + "LEUK" : {"code": "LEUK", "children": ["ALL", "CLL"], "parent": "TISSUE", "revocations": [], "precursors": [], "history": []}, + "MEL" : {"code": "MEL", "children": ["GMUCM", "SLL"], "parent": "TISSUE", "revocations": [], "precursors": [], "history": []}, + "PTCL" : {"code": "PTCL", "children": ["PCTLNOS"], "parent": "TNKL", "revocations": [], "precursors": [], "history": []}, + "PTCLNOS" : {"code": "PTCLNOS", "children": [], "parent": "PTCL", "revocations": [], "precursors": [], "history": []}, + "SEZS" : {"code": "SEZS", "children": ["FAKE_OLD_SS_CHILD"], "parent": "CTCL", "revocations": [], "precursors": [], "history": []}, + "SLL" : {"code": "SLL", "children": [], "parent": "MEL", "revocations": [], "precursors": [], "history": []}, + "TALL" : {"code": "TALL", "children": [], "parent": "ALL", "revocations": [], "precursors": [], "history": []}, + "TISSUE" : {"code": "TISSUE", "children": ["LEUK", "MEL", "TNKL"], "parent": "", "revocations": [], "precursors": [], "history": []}, + "TNKL" : {"code": "TNKL", "children": ["PTCL", "CTCL", "TNKL_CHILD"], "parent": "TISSUE", "revocations": [], "precursors": [], "history": []}, + "TNKL_CHILD" : {"code": "TNKL_CHILD", "children": [], "parent": "TNKL", "revocations": [], "precursors": [], "history": []}, + "TNKL_GRANDCHILD" : {"code": "TNKL_GRANDCHILD", "children": [], "parent": "TNKL_CHILD", "revocations": [], "precursors": [], "history": []} + } + + @classmethod + def get_latest_version(cls): + return { + "BLADDER" : {"code": "BLADDER", "children": ["URMM", "CLLSLL"], "parent": "TISSUE", "revocations": [], "precursors": [], "history": []}, + "BLL" : {"code": "BLL", "children": [], "parent": "LNM", "revocations": ["ALL"], "precursors": ["BALL"], "history": []}, + "CLLSLL" : {"code": "CLLSLL", "children": [], "parent": "BLADDER", "revocations": [], "precursors": ["SLL", "CLL"], "history": []}, + "DLL" : {"code": "DLL", "children": [], "parent": "LNM", "revocations": [], "precursors": [], "history": ["DALL"]}, + "FAKE_NEW_SS_CHILD" : {"code": "FAKE_NEW_SS_CHILD", "children": [], "parent": "SS", "revocations": [], "precursors": [], "history": [""]}, + "FAKE_OLD_SS_CHILD" : {"code": "FAKE_OLD_SS_CHILD", "children": [], "parent": "SS", "revocations": [], "precursors": [], "history": [""]}, + "LNM" : {"code": "LNM", "children": ["BLL"], "parent": "TISSUE", "revocations": [], "precursors": [], "history": []}, + "MTNN" : {"code": "MTNN", "children": ["SS", "PTCL", "MYCF"], "parent": "TISSUE", "revocations": [], "precursors": [], "history": []}, + "MYCF" : {"code": "MYCF", "children": [], "parent": "MTNN", "revocations": [], "precursors": ["CTCL"], "history": []}, + "PTCL" : {"code": "PTCL", "children": [], "parent": "MTNN", "revocations": ["PTCL"], "precursors": [], "history": ["PTCLNOS"]}, + "SS" : {"code": "SS", "children": ["FAKE_NEW_SS_CHILD", "FAKE_OLD_SS_CHILD"], "parent": "MTNN", "revocations": [], "precursors": [], "history": ["SEZS"]}, + "TLL" : {"code": "TLL", "children": [], "parent": "LNM", "revocations": ["ALL"], "precursors": ["TALL"], "history": []}, + "TISSUE" : {"code": "TISSUE", "children": ["MTNN", "BLADDER", "LNM"], "parent": "", "revocations": [], "precursors": [], "history": []}, + "TNKL_GRANDCHILD" : {"code": "TNKL_GRANDCHILD", "children": [], "parent": "TNKL_NEW_CHILD", "revocations": [], "precursors": [], "history": []}, + "TNKL_NEW_CHILD" : {"code": "TNKL_NEW_CHILD", "children": ["TNKL_GRANDCHILD"], "parent": "MTNN", "revocations": [], "precursors": [], "history": [""]}, + "TNKL_NEW_CHILD2" : {"code": "TNKL_NEW_CHILD2", "children": [], "parent": "MTNN", "revocations": [], "precursors": [], "history": [""]}, + "TNKL_CHILD2" : {"code": "TNKL_CHILD2", "children": ["TNKL_GRANDCHILD"], "parent": "MTNN", "revocations": [], "precursors": [], "history": ["TNKL_CHILD"]}, + "URMM" : {"code": "URMM", "children": [], "parent": "BLADDER", "revocations": ["GMUCM"], "precursors": [], "history": []} + } + +if __name__ == '__main__': + unittest.main()