Update fafb data version to 783 (#93)

* - update fafb data to v783
- retire SVD embeddings and morphology clusters

* ingest consolidated cell types

* fix tests

Author: arie-matsliah

README.md

@@ -17,24 +17,23 @@ We recommend using an environment manager such as [Poetry](https://python-poetry
 
 ```sh
 poetry install
-poetry shell
 ```
 
 ### Download and initialize the FlyWire connectome data (initially or upon version updates)
 ```bash
-./scripts/make_data.sh
+poetry run ./scripts/make_data.sh
 ```
 
 ## Run service locally
 
 ```bash
-./scripts/run_local.sh
+poetry run ./scripts/run_local.sh
 ```
 
 To run in [Flask debug mode](https://flask.palletsprojects.com/en/2.2.x/debugging/#the-built-in-debugger)
 
 ```sh
-./scripts/run_local_dev.sh
+poetry run ./scripts/run_local_dev.sh
 ```
 
 Navigate to [localhost:5000](http://localhost:5000)
@@ -48,11 +47,11 @@ Run service locally and click around in all pages
 ### Unit tests & code coverage (Required)
 
 ```sh
-./scripts/run_unit_tests.sh
+poetry run ./scripts/run_unit_tests.sh
 ```
 
 ## Linting / code formatting
 
 ```sh
-./scripts/lint.sh
+poetry run ./scripts/lint.sh
 ```

codex/blueprints/app.py

@@ -104,9 +104,11 @@ def stats():
         searched_for_root_id=can_be_flywire_root_id(filter_string),
         # If num results is small enough to pass to browser, pass it to allow copying root IDs to clipboard.
         # Otherwise it will be available as downloadable file.
-        root_ids_str=",".join([str(ddi) for ddi in filtered_root_id_list])
-        if len(filtered_root_id_list) <= MAX_NEURONS_FOR_DOWNLOAD
-        else [],
+        root_ids_str=(
+            ",".join([str(ddi) for ddi in filtered_root_id_list])
+            if len(filtered_root_id_list) <= MAX_NEURONS_FOR_DOWNLOAD
+            else []
+        ),
         filter_string=filter_string,
         hint=hint,
         data_versions=DATA_SNAPSHOT_VERSION_DESCRIPTIONS,
@@ -205,7 +207,6 @@ def render_neuron_list(
             "class",
             "sub_class",
             "cell_type",
-            "hemibrain_type",
             "hemilineage",
             "nt_type",
             "nerve",
@@ -227,9 +228,11 @@ def render_neuron_list(
         skeleton_thumbnail_urls=skeleton_thumbnail_urls,
         # If num results is small enough to pass to browser, pass it to allow copying root IDs to clipboard.
         # Otherwise it will be available as downloadable file.
-        root_ids_str=",".join([str(ddi) for ddi in sorted_search_result_root_ids])
-        if len(sorted_search_result_root_ids) <= MAX_NEURONS_FOR_DOWNLOAD
-        else [],
+        root_ids_str=(
+            ",".join([str(ddi) for ddi in sorted_search_result_root_ids])
+            if len(sorted_search_result_root_ids) <= MAX_NEURONS_FOR_DOWNLOAD
+            else []
+        ),
         num_items=len(sorted_search_result_root_ids),
         searched_for_root_id=can_be_flywire_root_id(filter_string),
         pagination_info=pagination_info,
@@ -256,11 +259,6 @@ def render_neuron_list(
             page_ids=page_ids,
             all_ids=sorted_search_result_root_ids,
         ),
-        non_uniform_hemibrain_types=neuron_db.non_uniform_values(
-            list_attr_key="hemibrain_type",
-            page_ids=page_ids,
-            all_ids=sorted_search_result_root_ids,
-        ),
     )
 
 
@@ -280,9 +278,6 @@ def _search_and_sort():
         output_sets=neuron_db.output_sets(),
         label_count_getter=lambda x: len(neuron_db.get_neuron_data(x)["label"]),
         nt_type_getter=lambda x: neuron_db.get_neuron_data(x)["nt_type"],
-        morphology_cluster_getter=lambda x: neuron_db.get_neuron_data(x)[
-            "morphology_cluster"
-        ],
         synapse_neuropil_count_getter=lambda x: len(
             neuron_db.get_neuron_data(x)["input_neuropils"]
         )
@@ -292,7 +287,6 @@ def _search_and_sort():
         + len(neuron_db.input_sets()[x]),
         similar_shape_cells_getter=neuron_db.get_similar_shape_cells,
         similar_connectivity_cells_getter=neuron_db.get_similar_connectivity_cells,
-        similar_embedding_cells_getter=neuron_db.get_similar_embedding_cells,
         connections_getter=lambda x: neuron_db.cell_connections(x),
         sort_by=sort_by,
     )
@@ -377,15 +371,12 @@ def download_search_results():
         "class",
         "sub_class",
         "cell_type",
-        "hemibrain_type",
         "hemilineage",
         "nerve",
         "connectivity_tag",
         "side",
         "input_synapses",
         "output_synapses",
-        "morphology_cluster",
-        "connectivity_cluster",
     ]
     data = [cols]
     for i in sorted_search_result_root_ids:
@@ -702,9 +693,9 @@ def path_length():
                 from_root_id = int(r[0])
                 for j, val in enumerate(r):
                     if j == 0:
-                        r[
-                            j
-                        ] = f'<a href="{url_for("app.search", filter_string="id == " + str(from_root_id))}">{neuron_db.get_neuron_data(from_root_id)["name"]}</a><br><small>{from_root_id}</small>'
+                        r[j] = (
+                            f'<a href="{url_for("app.search", filter_string="id == " + str(from_root_id))}">{neuron_db.get_neuron_data(from_root_id)["name"]}</a><br><small>{from_root_id}</small>'
+                        )
                     elif val > 0:
                         to_root_id = int(matrix[0][j])
                         if not min_syn_count:

codex/data/catalog.py

@@ -12,24 +12,18 @@
         "ach_avg",
         "oct_avg",
     ],
-    # LR matching
-    "lr_matching": [
-        "left_root_id",
-        "right_root_id",
-    ],
     # one row per id
     "classification": [
         "root_id",
         "flow",
         "super_class",
         "class",
         "sub_class",
-        "cell_type",
-        "hemibrain_type",
         "hemilineage",
         "side",
         "nerve",
     ],
+    "cell_types": ["root_id", "primary_type", "additional_type(s)"],
     # one row per id
     "cell_stats": [
         "root_id",
@@ -73,25 +67,21 @@
         "root_id",
         "scores",
     ],
-    # one row (or none) per id
-    "morphology_clusters": ["root_id", "morphology_cluster"],
-    # one row (or none) per id
-    "connectivity_clusters": ["root_id", "connectivity_cluster"],
 }
 
 
 def get_neurons_file_columns():
     return list(_CODEX_DATA_SCHEMA["neurons"])
 
 
-def get_lr_matching_file_columns():
-    return list(_CODEX_DATA_SCHEMA["lr_matching"])
-
-
 def get_classification_file_columns():
     return list(_CODEX_DATA_SCHEMA["classification"])
 
 
+def get_cell_types_file_columns():
+    return list(_CODEX_DATA_SCHEMA["cell_types"])
+
+
 def get_cell_stats_file_columns():
     return list(_CODEX_DATA_SCHEMA["cell_stats"])
 
@@ -114,11 +104,3 @@ def get_nblast_file_columns():
 
 def get_connectivity_tags_file_columns():
     return list(_CODEX_DATA_SCHEMA["connectivity_tags"])
-
-
-def get_morphology_clusters_columns():
-    return list(_CODEX_DATA_SCHEMA["morphology_clusters"])
-
-
-def get_connectivity_clusters_columns():
-    return list(_CODEX_DATA_SCHEMA["connectivity_clusters"])

codex/data/local_data_loader.py

@@ -3,7 +3,7 @@
 import gzip
 import os
 import pickle
-from datetime import datetime
+from datetime import datetime, UTC
 
 from codex.data.neuron_data_initializer import (
     initialize_neuron_data,
@@ -17,23 +17,20 @@
 DATA_ROOT_PATH = "static/data"
 NEURON_FILE_NAME = "neurons.csv.gz"
 CLASSIFICATION_FILE_NAME = "classification.csv.gz"
+CONSOLIDATED_CELL_TYPES_FILE_NAME = "consolidated_cell_types.csv.gz"
 CELL_STATS_ROWS = "cell_stats.csv.gz"
 CONNECTIONS_FILE_NAME = "connections.csv.gz"
 LABELS_FILE_NAME = "labels.csv.gz"
 COORDINATES_FILE_NAME = "coordinates.csv.gz"
 NBLAST_FILE_NAME = "nblast.csv.gz"
 CONNECTIVITY_TAGS_FILE_NAME = "connectivity_tags.csv.gz"
-MORPHOLOGY_CLUSTERS_FILE_NAME = "morphology_clusters.csv.gz"
-CONNECTIVITY_CLUSTERS_FILE_NAME = "connectivity_clusters.csv.gz"
-SVD_FILE_NAME = "svd.csv.gz"
-LR_MATCHING_FILE_NAME = "lr_matching.csv.gz"
 
 
 NEURON_DB_PICKLE_FILE_NAME = "neuron_db.pickle.gz"
 
-GCS_PICKLE_URL_TEMPLATE = "https://storage.googleapis.com/flywire-data/codex/data/{version}/neuron_db.pickle.gz"
+GCS_PICKLE_URL_TEMPLATE = "https://storage.googleapis.com/flywire-data/codex/data/fafb/{version}/neuron_db.pickle.gz"
 GCS_RAW_DATA_URL_TEMPLATE = (
-    "https://storage.googleapis.com/flywire-data/codex/data/{version}/{filename}"
+    "https://storage.googleapis.com/flywire-data/codex/data/fafb/{version}/{filename}"
 )
 
 
@@ -67,8 +64,8 @@ def _read_data(filename, with_timestamp=False):
         if os.path.exists(fname):
             rows = read_csv(fname)
             if with_timestamp:
-                return rows, datetime.utcfromtimestamp(
-                    os.path.getmtime(fname)
+                return rows, datetime.fromtimestamp(
+                    os.path.getmtime(fname), UTC
                 ).strftime("%Y-%m-%d")
             else:
                 return rows
@@ -80,6 +77,7 @@ def _read_data(filename, with_timestamp=False):
 
     neuron_rows = _read_data(NEURON_FILE_NAME)
     classification_rows = _read_data(CLASSIFICATION_FILE_NAME)
+    cell_type_rows = _read_data(CONSOLIDATED_CELL_TYPES_FILE_NAME)
     cell_stats_rows = _read_data(CELL_STATS_ROWS)
     connection_rows = _read_data(CONNECTIONS_FILE_NAME)
     label_rows, labels_file_timestamp = _read_data(
@@ -88,10 +86,6 @@ def _read_data(filename, with_timestamp=False):
     coordinate_rows = _read_data(COORDINATES_FILE_NAME)
     nblast_rows = _read_data(NBLAST_FILE_NAME)
     connectivity_tag_rows = _read_data(CONNECTIVITY_TAGS_FILE_NAME)
-    morphology_cluster_rows = _read_data(MORPHOLOGY_CLUSTERS_FILE_NAME)
-    connectivity_cluster_rows = _read_data(CONNECTIVITY_CLUSTERS_FILE_NAME)
-    svd_rows = _read_data(SVD_FILE_NAME)
-    lr_matching_rows = _read_data(LR_MATCHING_FILE_NAME)
 
     print(
         f" loading data from {data_file_path}:\n"
@@ -104,23 +98,21 @@ def _read_data(filename, with_timestamp=False):
     neuron_db = initialize_neuron_data(
         neuron_file_rows=neuron_rows,
         classification_rows=classification_rows,
+        cell_type_rows=cell_type_rows,
         cell_stats_rows=cell_stats_rows,
         connection_rows=connection_rows,
         label_rows=label_rows,
         labels_file_timestamp=labels_file_timestamp,
         coordinate_rows=coordinate_rows,
         nblast_rows=nblast_rows,
         connectivity_tag_rows=connectivity_tag_rows,
-        morphology_cluster_rows=morphology_cluster_rows,
-        connectivity_cluster_rows=connectivity_cluster_rows,
-        svd_rows=svd_rows,
-        lr_matching_rows=lr_matching_rows,
     )
     # free mem
     del neuron_rows
     del connection_rows
     del label_rows
     del coordinate_rows
+    del cell_type_rows
     return neuron_db