Send progress results to ldot

Author: “Carmel

app.py

@@ -8,7 +8,7 @@
 from new_ldot_workflows.b_2_get_qualtrics_links import add_individuals_to_survey
 from new_ldot_workflows.b_2_send_links_to_ldot import send_links_to_ldot
 from new_ldot_workflows.b_3_get_incomplete_subjects import get_incomplete_subjects
-# from new_ldot_workflows.b_4_get_individual_progress import get_individual_progress
+from new_ldot_workflows.b_4_get_individual_progress import get_individual_progress
 
 app = Flask(__name__)
 
@@ -41,9 +41,16 @@ def button1():
 
     ldot_study_id = ldot_vars.get("ldot_study_id")
     link_creation_eaid = ldot_vars.get("eaid_qualtrics_survey_link_creation_to_do_date")
-    link_completed_eaid = ldot_vars.get("eaid_qualtrics_survey_link_completed")
+    link_completed_eaid = ldot_vars.get("eaid_qualtrics_survey_link_creation_completed")
 
-    new_subjects_ids = get_new_subjects(ldot_study_id, link_creation_eaid, link_completed_eaid)
+    try:
+        new_subjects_ids = get_new_subjects(ldot_study_id, link_creation_eaid, link_completed_eaid)
+    except requests.RequestException as e:
+        return jsonify({"success": False, "message": f"Failed to fetch new subjects from LDOT: {e}"}), 502
+    except (KeyError, ValueError, TypeError) as e:
+        return jsonify({"success": False, "message": f"Unexpected LDOT response while fetching new subjects: {e}"}), 502
+    except Exception as e:
+        return jsonify({"success": False, "message": f"Unexpected error in button1: {e}"}), 500
 
     message = f"Found {len(new_subjects_ids)} new subjects in this study"
     return jsonify({"success": True, "message": message, "new_subject_ids": new_subjects_ids})
@@ -61,7 +68,10 @@ def button2():
         return jsonify({"success": False, "message": "Missing study_id in request"}), 400
 
     if not new_subject_ids:
-        return jsonify({"success": False, "message": "Missing new_subject_ids in request"}), 400
+        return jsonify({
+            "success": True,
+            "message": f"No new subjects to process",
+        })
 
     # Lookup study variables from STUDIES config
     study, ldot_vars, qualtrics_vars = get_study_settings(study_id)
@@ -78,9 +88,6 @@ def button2():
     directory_id = qualtrics_vars.get("directory_id")
     distribution_id = qualtrics_vars.get("distribution_id")
 
-    if not ldot_study_id:
-        return jsonify({"success": False, "message": f"Missing ldot_study_id for study_id: {study_id}"}), 400
-
     debug_inputs = {
         "study_id": study_id,
         "ldot_study_id": ldot_study_id,
@@ -120,9 +127,10 @@ def button3():
     eaid_survey_progress_completed = ldot_vars.get("eaid_survey_progress_completed")
 
     try:
-        result = f"Button 3 executed for study {study_id}"
         subjects_not_completed_survey = get_incomplete_subjects(ldot_study_id, eaid_survey_invitation_completed, eaid_survey_progress_completed)
-        return jsonify({"success": True, "message": result, "subject_ids": subjects_not_completed_survey})
+        print("Subjects not completed survey:", subjects_not_completed_survey)
+        message = f"Found {len(subjects_not_completed_survey)} subjects who have not completed the survey"
+        return jsonify({"success": True, "message": message, "subject_ids": subjects_not_completed_survey})
     except Exception as e:
         return jsonify({"success": False, "message": str(e)})
 
@@ -137,17 +145,15 @@ def button4():
     if not study_id:
         return jsonify({"success": False, "message": "Missing study_id in request"}), 400
 
-    # if not subject_ids:
-    #     return jsonify({"success": False, "message": "Missing subject_ids in request"}), 400
-
-    # study, vars = get_study_settings(study_id)
-    # if not study:
-    #     return jsonify({"success": False, "message": f"Unknown study_id: {study_id}"}), 400
+    study, ldot_vars, qualtrics_vars = get_study_settings(study_id)
 
-    # qualtrics_survey_id = vars.get("qualtrics_survey_id")
-    # embedded_data_field = vars.get("embedded_data_field")
+    ldot_study_id = ldot_vars.get("ldot_study_id")
+    id_deelnemer_entity = ldot_vars.get("id_deelnemer_entity")
+    id_location = ldot_vars.get("id_location")
+    qualtrics_survey_id = qualtrics_vars.get("qualtrics_survey_id")
+    embedded_data_field = qualtrics_vars.get("embedded_data_field")
 
-    # participant_to_progress_dict = get_individual_progress(subject_ids, embedded_data_field, qualtrics_survey_id)
+    participant_to_progress_dict = get_individual_progress(ldot_study_id, id_deelnemer_entity, id_location, subject_ids, embedded_data_field, qualtrics_survey_id)
 
 
     # return jsonify({

new_ldot_workflows/__pycache__/b_1_get_new_subjects.cpython-311.pyc

@@ -7,9 +7,20 @@
 CLIENT_SECRET = config["client_secret"]
 LDOT_API_URL = config["LDOT_API_URL"]
 
+
+def _response_snippet(response: requests.Response, limit: int = 300) -> str:
+    text = response.text.strip().replace("\n", " ")
+    return text[:limit]
+
+
+def _format_response_error(prefix: str, response: requests.Response) -> str:
+    return f"{prefix} (status {response.status_code}): {_response_snippet(response)}"
+
 def get_new_subjects(study_id: str, eaid_qualtrics_survey_link_creation_to_do_date: str, eaid_qualtrics_survey_link_completed: str) -> list:
     """Get subjects that have not yet been added to Qualtrics by checking their event actions"""
 
+    print(f"Inputs into get_new_subjects: study_id={study_id}, eaid_qualtrics_survey_link_creation_to_do_date={eaid_qualtrics_survey_link_creation_to_do_date}, eaid_qualtrics_survey_link_completed={eaid_qualtrics_survey_link_completed}")
+
     response = requests.post(
         "https://accware.memic.maastrichtuniversity.nl/ldot_identity_server/connect/token",
         data={
@@ -19,7 +30,16 @@ def get_new_subjects(study_id: str, eaid_qualtrics_survey_link_creation_to_do_da
         }
     )
 
-    token = response.json()["access_token"]
+    response.raise_for_status()
+
+    try:
+        token_payload = response.json()
+    except ValueError as e:
+        raise ValueError(_format_response_error("LDOT token endpoint returned invalid JSON", response)) from e
+
+    token = token_payload.get("access_token")
+    if not token:
+        raise ValueError(_format_response_error("LDOT token endpoint did not include an access_token", response))
 
     headers={"accept": "application/json",
             "Authorization": f"Bearer {token}"
@@ -32,7 +52,11 @@ def get_new_subjects(study_id: str, eaid_qualtrics_survey_link_creation_to_do_da
     )
 
     response.raise_for_status()
-    payload = response.json()
+
+    try:
+        payload = response.json()
+    except ValueError as e:
+        raise ValueError(_format_response_error("LDOT creation-to-do lookup returned invalid JSON", response)) from e
     study_event_actions = payload.get("Data", {}).get("StudyEventActions", [])
 
     subjects_link_creation_to_do = set([
@@ -41,29 +65,33 @@ def get_new_subjects(study_id: str, eaid_qualtrics_survey_link_creation_to_do_da
         if action.get("SubjectGuid")
     ])
 
-    # print(f"Subjects with link creation to do: {subjects_link_creation_to_do}")
+    print(f"Subjects with link creation to do: {subjects_link_creation_to_do}")
 
     # Get subjects that have link completed date
     response = requests.get(
         f"https://accware.memic.maastrichtuniversity.nl/memic_ldot_api/api/v1.1/{study_id}/Action/{eaid_qualtrics_survey_link_completed}",
         headers=headers
     )
     response.raise_for_status()
-    payload = response.json()
+
+    try:
+        payload = response.json()
+    except ValueError as e:
+        raise ValueError(_format_response_error("LDOT completed lookup returned invalid JSON", response)) from e
     study_event_actions_completed = payload.get("Data", {}).get("StudyEventActions", [])
     subjects_link_completed = set([
         action["SubjectGuid"]
         for action in study_event_actions_completed
         if action.get("SubjectGuid")
     ])
 
-    # print(f"Subjects with link completed: {subjects_link_completed}")
+    print(f"Subjects with link completed: {subjects_link_completed}")
 
     # Filter out subjects that have already completed the link
     new_subjects = subjects_link_creation_to_do - subjects_link_completed
 
-    # print(f"New subjects: {new_subjects}")
-    return list(new_subjects)
+    print(f"New subjects: {new_subjects}")
+    return list(new_subjects) if new_subjects else []
 
 
 

new_ldot_workflows/__pycache__/b_3_get_incomplete_subjects.cpython-311.pyc

@@ -11,6 +11,8 @@
 def get_incomplete_subjects(study_id: str, eaid_survey_invitation_completed: str = None, eaid_survey_progress_completed: str = None) -> list:
     """Get subjects that have not yet completed the survey by checking their event actions"""
 
+    print("Inputs to get_incomplete_subjects: study_id={}, eaid_survey_invitation_completed={}, eaid_survey_progress_completed={}".format(study_id, eaid_survey_invitation_completed, eaid_survey_progress_completed))
+
     response = requests.post(
         "https://accware.memic.maastrichtuniversity.nl/ldot_identity_server/connect/token",
         data={
@@ -35,8 +37,9 @@ def get_incomplete_subjects(study_id: str, eaid_survey_invitation_completed: str
 
     subjects_with_survey_invitation_completed = set()
 
-    for action in result.json().get("Data", {}).get("StudyEventActions", []):
-        subjects_with_survey_invitation_completed.add(action["SubjectGuid"])
+    # print(result.json())
+    # for action in result.json().get("Data", {}).get("StudyEventActions", []):
+    #     subjects_with_survey_invitation_completed.add(action["SubjectGuid"])
 
     # print(f"Subjects with survey invitation completed: {subjects_with_survey_invitation_completed}")
 
@@ -45,15 +48,20 @@ def get_incomplete_subjects(study_id: str, eaid_survey_invitation_completed: str
         headers=headers
     )
     result.raise_for_status()
+    print(result.json())
 
-    subjects_with_survey_progress_completed = set()
-    for action in result.json().get("Data", {}).get("StudyEventActions", []):
-        subjects_with_survey_progress_completed.add(action["SubjectGuid"])
+    # subjects_with_survey_progress_completed = set()
+    # for action in result.json().get("Data", {}).get("StudyEventActions", []):
+    #     subjects_with_survey_progress_completed.add(action["SubjectGuid"])
 
     # print(f"Subjects with survey progress completed: {subjects_with_survey_progress_completed}")
 
-    incomplete_subjects = subjects_with_survey_invitation_completed - subjects_with_survey_progress_completed
-    return list(incomplete_subjects)
+    # incomplete_subjects = subjects_with_survey_invitation_completed - subjects_with_survey_progress_completed
+    # return list(incomplete_subjects)
 
 if __name__ == "__main__":
-    print(get_incomplete_subjects("5c9c6a47-c8d7-8142-a8c8-ccdcb8a8044b", "9f09f2cf-32cc-844d-9a41-7a44accd3dfd", '120d298d-9aa9-084b-abc6-432148db0e10'))
+
+    ldot_study_id = "5c9c6a47-c8d7-8142-a8c8-ccdcb8a8044b"
+    eaid_survey_invitation_completed = "337ecba2-3dd3-1141-8c38-6cffdcfbd7eb"
+    eaid_survey_progress_completed = "120d298d-9aa9-084b-abc6-432148db0e10"
+    print(get_incomplete_subjects(ldot_study_id, eaid_survey_invitation_completed, eaid_survey_progress_completed))

new_ldot_workflows/__pycache__/b_4_get_individual_progress.cpython-311.pyc

@@ -1,9 +1,24 @@
 import requests
-from qualtrics_settings import BASE_URL, HEADERS, QualtricsAPIError
 import time
 import zipfile
 import io
 import pandas as pd
+import json
+
+with open("new_ldot_workflows/qualtrics_config.json") as f:
+    config = json.load(f)
+QUALTRICS_BASE_URL = config["QUALTRICS_BASE_URL"]
+HEADERS = config["HEADERS"]
+
+with open("new_ldot_workflows/ldot_config.json") as f:
+    config = json.load(f)
+CLIENT_ID = config["client_id"]
+CLIENT_SECRET = config["client_secret"]
+LDOT_API_URL = config["LDOT_API_URL"]
+
+class QualtricsAPIError(Exception):
+    """Custom exception for Qualtrics API errors"""
+    pass
 
 def create_data_export(survey_id: str, incomplete_bool: bool = True) -> str:
     """Create a data export request for a given survey.
@@ -17,7 +32,7 @@ def create_data_export(survey_id: str, incomplete_bool: bool = True) -> str:
         str: The ID of the data export request.
     """
     print("Creating data export request... ")
-    endpoint = f"{BASE_URL}/surveys/{survey_id}/export-responses"
+    endpoint = f"{QUALTRICS_BASE_URL}/surveys/{survey_id}/export-responses"
     payload = {
         "format": "csv",
         "exportResponsesInProgress": incomplete_bool
@@ -51,7 +66,7 @@ def check_export_progress(request_id: str, survey_id: str, returns: str) -> str|
         str: The percent complete if the request is still processing, or the file ID if complete.
     """
     print("Checking on export progress... ")
-    endpoint = f"{BASE_URL}/surveys/{survey_id}/export-responses/{request_id}"
+    endpoint = f"{QUALTRICS_BASE_URL}/surveys/{survey_id}/export-responses/{request_id}"
     try:
         result = requests.get(endpoint, headers=HEADERS)
         result.raise_for_status()
@@ -84,7 +99,7 @@ def download_export(file_id: str, survey_id: str) -> pd.DataFrame:
         pd.DataFrame: The survey response data as a DataFrame.
     """
     print("Downloading export file... ")
-    endpoint = f"{BASE_URL}/surveys/{survey_id}/export-responses/{file_id}/file"
+    endpoint = f"{QUALTRICS_BASE_URL}/surveys/{survey_id}/export-responses/{file_id}/file"
     try:
         download = requests.get(
             endpoint,
@@ -133,6 +148,7 @@ def check_inputs_validity(participant_study_id: str, embedded_data_field: str, s
         raise ValueError("Invalid participant_study_id. It should be a non-empty string.")
     return True
 
+
 def get_responses_as_df(survey_id: str, incomplete_bool: bool) -> pd.DataFrame:
     """Wrapper function to request export, wait for it, download it, and convert to DataFrame.
 
@@ -157,13 +173,49 @@ def get_responses_as_df(survey_id: str, incomplete_bool: bool) -> pd.DataFrame:
     df = download_export(file_id, survey_id)
     return df
 
-def get_individual_progress(participant_study_ids: list, embedded_data_field: str, survey_id: str) -> float:
+
+def subject_id_to_study_identifier(ldot_study_id: str, id_deelnemer_entity: str, id_location: str, subject_ids: list) -> str:
+    """Post the Qualtrics links back to Ldot for new subjects"""
+
+    response = requests.post(
+        "https://accware.memic.maastrichtuniversity.nl/ldot_identity_server/connect/token",
+        data={
+            "grant_type": "client_credentials",
+            "client_id": CLIENT_ID,
+            "client_secret": CLIENT_SECRET
+        }
+    )
+    token = response.json()["access_token"]
+    headers={"accept": "application/json",
+            "Authorization": f"Bearer {token}"
+            }
+
+    subject_id_to_study_identifier_dict = {}
+    for subject_id in subject_ids:
+        # Populate the Qualtrics link in Ldot for the subject
+        response = requests.post(
+            f"https://accware.memic.maastrichtuniversity.nl/memic_ldot_api/api/v1.1/{ldot_study_id}/Subject/",
+            headers=headers,
+            json = {
+                "subjectGuid": subject_id,
+                "entityId": id_deelnemer_entity,
+                "locationSubjectId": id_location,
+                "doOverwriteNulls": False,
+            }
+        )
+        subject_id_to_study_identifier_dict[subject_id] = response.json().get("Data", {}).get("Subject").get("RegistrationId")
+
+
+    return subject_id_to_study_identifier_dict
+
+
+def get_individual_progress(ldot_study_id: str, id_deelnemer_entity: str, id_location: str, subject_ids: list, embedded_data_field: str, survey_id: str) -> float:
     """Wrapper function that gets the individual progress of a participant in a survey."""
     # Check for invalid inputs
+    check_inputs_validity(subject_ids[0], embedded_data_field, survey_id) # TODO: Check this
 
     participant_to_progress_dict = {}
-    
-    check_inputs_validity(participant_study_ids[0], embedded_data_field, survey_id)
+    subject_id_to_study_identifier_dict = subject_id_to_study_identifier(ldot_study_id, id_deelnemer_entity, id_location, subject_ids)
 
     completed_responses_df = get_responses_as_df(survey_id, incomplete_bool=False)
     incompleted_responses_df = get_responses_as_df(survey_id, incomplete_bool=True)
@@ -172,21 +224,31 @@ def get_individual_progress(participant_study_ids: list, embedded_data_field: st
     if not embedded_data_field in df.columns:
         raise QualtricsAPIError(f"Embedded data field '{embedded_data_field}' not found in survey data. Please check the field name.")
 
-    for participant_study_id in participant_study_ids:
-        individual_progress = df[df[embedded_data_field] == participant_study_id]
+    for subject_id, study_identifier in subject_id_to_study_identifier_dict.items():
+        individual_progress = df[df[embedded_data_field] == study_identifier]
+        print("This is the individual progress for subject_id {}, study_identifier {}: {}".format(subject_id, study_identifier, individual_progress))
         if not individual_progress.empty:
             individual_progress = individual_progress["Progress"].values[0]
-            participant_to_progress_dict[participant_study_id] = individual_progress
+            participant_to_progress_dict[subject_id] = individual_progress
         else:
-            participant_to_progress_dict[participant_study_id] = None  # or some indicator that the participant ID was not found in the data
+            participant_to_progress_dict[subject_id] = 0  # Indicator that the subject ID was not found in the data
 
     return participant_to_progress_dict
 
 if __name__ == "__main__":
-    # Example usage
-    participant_study_ids=["11111TEST11111", "22222TEST22222", "33333TEST33333"]
+    # # Example usage
     survey_id="SV_efCMOg6wHU0T8ii"
     embedded_data_field = "study_id_child"
 
-    participant_to_progress_dict = get_individual_progress(participant_study_ids, embedded_data_field, survey_id)
-    print(participant_to_progress_dict)
+
+    ldot_study_id = "5c9c6a47-c8d7-8142-a8c8-ccdcb8a8044b"
+    subject_ids = ["352fb9d8-962f-4735-9fc7-7b4e18109a51"]
+    id_deelnemer_entity = "7f61b810-00ed-1d41-8a33-4164f25ebad0"
+    id_location = "427f304f-9d95-44f5-8f7b-d6a1ce1db293"
+    embedded_data_field = "study_id_child"
+
+    participant_to_progress_dict = get_individual_progress(ldot_study_id, id_deelnemer_entity, id_location, subject_ids, embedded_data_field, survey_id)
+    print(participant_to_progress_dict)
+
+
+    # subject_id_to_study_identifier_dict = subject_id_to_study_identifier(ldot_study_id, id_deelnemer_entity, id_location, subject_ids)