Merge pull request #119 from datakind/develop

fix: increased api limits

Author: Mesh-ach

src/webapp/databricks.py

@@ -206,7 +206,6 @@ def fetch_table_data(
         inst_name: str,
         table_name: str,
         warehouse_id: str,
-        limit: int = 1000,
     ) -> List[Dict[str, Any]]:
         """
         Executes a SELECT * query on the specified table within the given catalog and schema,
@@ -226,7 +225,7 @@ def fetch_table_data(
         fully_qualified_table = (
             f"`{catalog_name}`.`{schema_name}_silver`.`{table_name}`"
         )
-        sql_query = f"SELECT * FROM {fully_qualified_table} LIMIT {limit}"
+        sql_query = f"SELECT * FROM {fully_qualified_table}"
 
         try:
             # Execute the SQL statement

src/webapp/routers/data.py

@@ -1057,7 +1057,6 @@ def get_top_features(
             inst_name=f"{query_result[0][0].name}",
             table_name=f"sample_inference_{run_id}_features_with_most_impact",
             warehouse_id=env_vars["SQL_WAREHOUSE_ID"],
-            limit=500,
         )
 
         return rows
@@ -1101,7 +1100,6 @@ def get_support_overview(
             inst_name=f"{query_result[0][0].name}",
             table_name=f"sample_inference_{run_id}_support_overview",
             warehouse_id=env_vars["SQL_WAREHOUSE_ID"],
-            limit=500,
         )
 
         return rows
@@ -1144,7 +1142,6 @@ def get_training_support_overview(
             inst_name=f"{query_result[0][0].name}",
             table_name=f"sample_training_{run_id}_support_overview",
             warehouse_id=env_vars["SQL_WAREHOUSE_ID"],
-            limit=500,
         )
 
         return rows
@@ -1187,7 +1184,6 @@ def get_feature_value(
             inst_name=f"{query_result[0][0].name}",
             table_name=f"sample_inference_{run_id}_shap_feature_importance",
             warehouse_id=env_vars["SQL_WAREHOUSE_ID"],
-            limit=500,
         )
 
         return rows
@@ -1230,7 +1226,6 @@ def get_confusion_matrix(
             inst_name=f"{query_result[0][0].name}",
             table_name=f"sample_training_{run_id}_confusion_matrix",
             warehouse_id=env_vars["SQL_WAREHOUSE_ID"],
-            limit=500,
         )
 
         return rows
@@ -1273,7 +1268,6 @@ def get_roc_curve(
             inst_name=f"{query_result[0][0].name}",
             table_name=f"sample_training_{run_id}_roc_curve",
             warehouse_id=env_vars["SQL_WAREHOUSE_ID"],
-            limit=500,
         )
 
         return rows

src/webapp/validation_schemas/pdp_schema_extension.json

@@ -575,7 +575,7 @@
               "checks": []
             },
             "core_competency_completed": {
-              "dtype": "float64",
+              "dtype": "category",
               "coerce": true,
               "nullable": true,
               "required": true,
@@ -599,23 +599,23 @@
               "checks": []
             },
             "course_instructor_employment_status": {
-              "dtype": "float64",
+              "dtype": "category",
               "coerce": true,
               "nullable": true,
               "required": true,
               "aliases": [],
               "checks": []
             },
             "course_instructor_rank": {
-              "dtype": "float64",
+              "dtype": "category",
               "coerce": true,
               "nullable": true,
               "required": false,
               "aliases": [],
               "checks": []
             },
             "credential_engine_identifier": {
-              "dtype": "float64",
+              "dtype": "string",
               "coerce": true,
               "nullable": true,
               "required": true,
@@ -671,7 +671,7 @@
               "checks": []
             },
             "institution_id": {
-              "dtype": "float64",
+              "dtype": "string",
               "coerce": true,
               "nullable": true,
               "required": true,