Merge branch 'CONP-PCNO:master' into master

emmetaobrien · web-flow · commit a3800a303585 · 2026-05-28T14:33:15.000-04:00
diff --git a/.gitmodules b/.gitmodules
@@ -624,3 +624,30 @@
 	path = projects/Calgary-Preschool-MRI-Dataset
 	url = https://github.com/CONP-PCNO/Calgary-Preschool-MRI-Dataset
 	branch = main
+[submodule "projects/Participant_level_contrast_maps"]
+	path = projects/Participant_level_contrast_maps
+	url = https://github.com/conp-bot/conp-dataset-Participant_level_contrast_maps.git
+[submodule "projects/MICA-PNI_Precision_NeuroImaging_and_Connectomics"]
+	path = projects/MICA-PNI_Precision_NeuroImaging_and_Connectomics
+	url = https://github.com/conp-bot/conp-dataset-MICA-PNI_Precision_NeuroImaging_and_Connectomics.git
+[submodule "projects/Relational_and_Item-Specific_Encoding_(RISE)"]
+	path = projects/Relational_and_Item-Specific_Encoding_(RISE)
+	url = https://github.com/conp-bot/conp-dataset-Relational_and_Item-Specific_Encoding.git
+[submodule "projects/NiMARE_Neuroimaging_Meta-Analysis_Research_Environment"]
+	path = projects/NiMARE_Neuroimaging_Meta-Analysis_Research_Environment
+	url = https://github.com/conp-bot/conp-dataset-NiMARE_Neuroimaging_Meta-Analysis_Research_Environment.git
+[submodule "projects/3-step_CPCA"]
+	path = projects/3-step_CPCA
+	url = https://github.com/conp-bot/conp-dataset-3-step_CPCA.git
+[submodule "projects/PiDose"]
+	path = projects/PiDose
+	url = https://github.com/conp-bot/conp-dataset-PiDose.git
+[submodule "projects/conp-dataset-Expression-based_polygenic_score_from_the_amygdala_5HTT_gene_network"]
+	path = projects/conp-dataset-Expression-based_polygenic_score_from_the_amygdala_5HTT_gene_network
+	url = https://github.com/conp-bot/conp-dataset-Expression-based_polygenic_score_from_the_amygdala_5HTT_gene_networ.git
+[submodule "projects/MRI_data_for_Stress-inducible_phosphoprotein_1_HOP_STI1_STIP1_regulates_the_spre"]
+	path = projects/MRI_data_for_Stress-inducible_phosphoprotein_1_HOP_STI1_STIP1_regulates_the_spre
+	url = https://github.com/conp-bot/conp-dataset-MRI_data_for_Stress-inducible_phosphoprotein_1_HOP_STI1_STIP1_regul.git
+[submodule "projects/Neural_capacity_limits_on_the_responses_to_memory_interference_during_working_me"]
+	path = projects/Neural_capacity_limits_on_the_responses_to_memory_interference_during_working_me
+	url = https://github.com/conp-bot/conp-dataset-Neural_capacity_limits_on_the_responses_to_memory_interference_duri.git
diff --git a/projects/3-step_CPCA b/projects/3-step_CPCA
@@ -0,0 +1 @@
+Subproject commit a318d92b2fee5c25ce6171843a482afcc8d4e1b3
diff --git a/projects/MICA-PNI_Precision_NeuroImaging_and_Connectomics b/projects/MICA-PNI_Precision_NeuroImaging_and_Connectomics
@@ -0,0 +1 @@
+Subproject commit e69092864acf3719609b5bed53b42ff68d7674d1
diff --git a/projects/MRI_data_for_Stress-inducible_phosphoprotein_1_HOP_STI1_STIP1_regulates_the_spre b/projects/MRI_data_for_Stress-inducible_phosphoprotein_1_HOP_STI1_STIP1_regulates_the_spre
@@ -0,0 +1 @@
+Subproject commit 698550de8cc37e43eef9699f7aeab418e3a0e3a2
diff --git a/projects/Neural_capacity_limits_on_the_responses_to_memory_interference_during_working_me b/projects/Neural_capacity_limits_on_the_responses_to_memory_interference_during_working_me
@@ -0,0 +1 @@
+Subproject commit a1f1a6b942e5503fc3405efe1615c1d6c7e0f1a0
diff --git a/projects/NiMARE_Neuroimaging_Meta-Analysis_Research_Environment b/projects/NiMARE_Neuroimaging_Meta-Analysis_Research_Environment
@@ -0,0 +1 @@
+Subproject commit 1b91e67e4d0b79c23a312029b8f191d7e042f9cf
diff --git a/projects/Participant_level_contrast_maps b/projects/Participant_level_contrast_maps
@@ -0,0 +1 @@
+Subproject commit 2dad17f7b1ac25cce665557e975fd18331a3a148
diff --git a/projects/PiDose b/projects/PiDose
@@ -0,0 +1 @@
+Subproject commit 1b8edf29123b608a5e9c15a3d329a20133d92375
diff --git a/projects/Relational_and_Item-Specific_Encoding_(RISE) b/projects/Relational_and_Item-Specific_Encoding_(RISE)
@@ -0,0 +1 @@
+Subproject commit f9b3e84da250c13a6b6a58d9f0ebda60828b0690
diff --git a/projects/conp-dataset-Expression-based_polygenic_score_from_the_amygdala_5HTT_gene_network b/projects/conp-dataset-Expression-based_polygenic_score_from_the_amygdala_5HTT_gene_network
@@ -0,0 +1 @@
+Subproject commit a553ec900253545555b6459410173260d48f6af3
diff --git a/scripts/Crawlers/BaseCrawler.py b/scripts/Crawlers/BaseCrawler.py
@@ -2,7 +2,8 @@
 import json
 import os
 import re
-import shutil
+import subprocess
+from email.message import EmailMessage
 
 import git
 import requests
@@ -11,7 +12,6 @@
 from scripts.Crawlers.constants import DATS_FIELDS
 from scripts.Crawlers.constants import LICENSE_CODES
 from scripts.Crawlers.constants import MODALITIES
-from scripts.Crawlers.constants import NO_ANNEX_FILE_PATTERNS
 from scripts.Crawlers.constants import REQUIRED_DATS_FIELDS
 
 
@@ -84,6 +84,7 @@ def __init__(self, github_token, config_path, verbose, force, no_pr, basedir):
         self.git = git
         self.datalad = api
         self.no_pr = no_pr
+        self.config = self._load_config()
         if self.verbose:
             print(f"Using base directory {self.basedir}")
 
@@ -239,6 +240,16 @@ def run(self):
         if not. If dataset is already existing locally, verify if dataset needs updating
         with update_if_necessary() and update if so
         """
+        # Fetch latest remote branches to correctly identify existing datasets
+        try:
+            if self.verbose:
+                print("Fetching latest remote branches...")
+            self.repo.remotes.origin.fetch()
+            if self.verbose:
+                print("Fetch completed.")
+        except Exception as e:
+            print(f"Warning: Failed to fetch remote branches: {e}")
+            print("Continuing with cached remote branch information...")
         dataset_description_list = self.get_all_dataset_description()
         for dataset_description in dataset_description_list:
             try:
@@ -247,84 +258,21 @@ def run(self):
                 dataset_rel_dir = os.path.join("projects", clean_title)
                 dataset_dir = os.path.join(self.basedir, dataset_rel_dir)
                 d = self.datalad.Dataset(dataset_dir)
-                if branch_name not in self.repo.remotes.origin.refs:  # New dataset
-                    self.repo.git.checkout("-b", branch_name)
-                    repo_title = ("conp-dataset-" + dataset_description["title"])[0:100]
-                    try:
-                        d.create()
-                        r = d.create_sibling_github(
-                            repo_title,
-                            name="origin",
-                            github_login=self.github_token,
-                            github_passwd=self.github_token,
-                        )
-                    except Exception as error:
-                        # handle the exception
-                        print("An exception occurred:", error)
-
-                    # Add github token to dataset origin remote url
-                    try:
-                        origin = self.repo.remote("origin")
-                        origin_url = next(origin.urls)
-                        if "@" not in origin_url:
-                            origin.set_url(
-                                origin_url.replace(
-                                    "https://",
-                                    "https://" + self.github_token + "@",
-                                ),
-                            )
-                    except git.exc.NoSuchPathError:
-                        pass
-
-                    self._add_github_repo_description(repo_title, dataset_description)
-                    for pattern in NO_ANNEX_FILE_PATTERNS:
-                        d.no_annex(pattern)
-                    self.add_new_dataset(dataset_description, dataset_dir)
-
-                    # Create DATS.json if it exists in directory and 1 level deep subdir
-                    dats_path: str = os.path.join(dataset_dir, "DATS.json")
-                    if existing_dats_path := self._check_file_present(
-                        dataset_dir, "dats.json"
-                    ):
-                        if self.verbose:
-                            print(f"Found existing DATS.json at {existing_dats_path}")
-                        if existing_dats_path != dats_path:
-                            shutil.copy(existing_dats_path, dats_path)
-                        self._add_source_data_submodule_if_derived_from_conp_dataset(
-                            dats_path, dataset_dir
-                        )
-                    else:
-                        self._create_new_dats(
-                            dataset_dir,
-                            dats_path,
-                            dataset_description,
-                            d,
-                        )
-                    # Move the logo into the root directory if found in 1 level deep subdir
-                    logo_path = os.path.join(dataset_dir, "logo.png")
-                    if existing_logo_path := self._check_file_present(
-                        dataset_dir, "logo.png"
-                    ):
-                        if self.verbose:
-                            print(f"Found logo at {existing_logo_path}")
-                        if existing_logo_path != logo_path:
-                            os.rename(existing_logo_path, logo_path)
-
-                    # Create README.md if it doesn't exist
-                    if not os.path.isfile(os.path.join(dataset_dir, "README.md")):
-                        readme = self.get_readme_content(dataset_description)
-                        self._create_readme(
-                            readme, os.path.join(dataset_dir, "README.md")
-                        )
-                    d.save()
-                    d.publish(to="origin")
-                    self.repo.git.submodule(
-                        "add",
-                        r[0][1].replace(self.github_token + "@", ""),
+                # using remote_head
+                remote_branch_names = [
+                    ref.remote_head for ref in self.repo.remotes.origin.refs
+                ]
+                if branch_name not in remote_branch_names:  # New dataset
+                    self._notify_new_dataset(
+                        dataset_description,
+                        branch_name,
                         dataset_rel_dir,
                     )
-                    modified = True
-                    commit_msg = "Created " + dataset_description["title"]
+                    print(
+                        f"New dataset detected: {dataset_description['title']}. "
+                        "Notification sent; skipping automatic DataLad creation."
+                    )
+                    continue
                 else:  # Dataset already existing locally
                     try:
                         # Try normal checkout first
@@ -345,11 +293,25 @@ def run(self):
                             raise
 
                     try:
-                        self.repo.git.merge("-n", "--no-verify", "master")
+                        self.repo.git.execute(
+                            [
+                                "git",
+                                "-c",
+                                "filter.annex.process=",
+                                "merge",
+                                "-n",
+                                "--no-verify",
+                                "master",
+                            ]
+                        )
                     except Exception as e:
                         print(f"Error while merging master into {branch_name}: {e}")
                         print("Skipping this dataset")
-                        self.repo.git.merge("--abort")
+                        try:
+                            self.repo.git.merge("--abort")
+                        except git.exc.GitCommandError as abort_error:
+                            if "MERGE_HEAD missing" not in str(abort_error):
+                                raise
                         try:
                             # Use the same safe checkout to go back to master
                             self.repo.git.checkout("-f", "master")
@@ -435,6 +397,47 @@ def run(self):
                 else:
                     raise
 
+    def _load_config(self):
+        if not self.config_path or not os.path.isfile(self.config_path):
+            return {}
+        with open(self.config_path) as f:
+            return json.load(f)
+
+    def _notify_new_dataset(self, dataset_description, branch_name, dataset_rel_dir):
+        recipient = self.config.get("notify_email", "wangshen.mcin@gmail.com")
+        from_email = self.config.get("from_email", "conp-bot@localhost")
+        sendmail_path = self.config.get("sendmail_path", "/usr/sbin/sendmail")
+
+        msg = EmailMessage()
+        msg["To"] = recipient
+        msg["From"] = from_email
+        msg["Subject"] = (
+            "[CONP crawler] New dataset needs manual DataLad import: "
+            f"{dataset_description['title']}"
+        )
+        msg.set_content(
+            "\n".join(
+                [
+                    "The CONP crawler found a dataset that does not have an origin branch yet.",
+                    "",
+                    "Automatic DataLad repository creation is disabled on this host.",
+                    "Please import this dataset manually.",
+                    "",
+                    f"Title: {dataset_description['title']}",
+                    f"Identifier: {dataset_description.get('identifier', '')}",
+                    f"Version: {dataset_description.get('version', '')}",
+                    f"Branch: {branch_name}",
+                    f"Project path: {dataset_rel_dir}",
+                ]
+            )
+        )
+
+        subprocess.run(
+            [sendmail_path, "-t"],
+            input=msg.as_bytes(),
+            check=True,
+        )
+
     def _add_github_repo_description(self, repo_title, dataset_description):
         url = "https://api.github.com/repos/{}/{}".format(
             self.username,
