dags: Split description out of "Status" column

Implements: AP-684

Author: awilfox

mokelumne/dags/gen_llm_image_descriptions.py

@@ -33,7 +33,7 @@
 logger = logging.getLogger(__name__)
 
 
-RunStatus = namedtuple('RunStatus', ('tind_id', 'status', 'path'))
+RunStatus = namedtuple('RunStatus', ('tind_id', 'status', 'description', 'path'))
 
 SUPPORTED_IMAGE_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
 """The supported image MIME types we will fetch."""
@@ -213,15 +213,16 @@ def fetch_image_to_record_directory(run_id: str, fetcher: ImageFetcher,
                     return RunStatus(
                         tind_id=tind_id,
                         path="",
-                        status=f"skipped: Unsupported file type {file_md.get('mime')}",
+                        status="skipped",
+                        description=f"Unsupported file type {file_md.get('mime')}",
                     )
 
                 path = str(fetcher.fetch_one_image_for_record(tind_id, run_id))
             except Exception as ex:  # pylint: disable=broad-exception-caught
                 logger.warning("Fetcher encountered exception", exc_info=ex)
-                return RunStatus(tind_id=tind_id, status=f'failed: {str(ex)}', path='')
+                return RunStatus(tind_id=tind_id, status="failed", description=str(ex), path="")
 
-            return RunStatus(tind_id=tind_id, status="fetched", path=path)
+            return RunStatus(tind_id=tind_id, status="fetched", description="", path=path)
 
         @task
         def write_status_to_fetched_csv(
@@ -233,13 +234,16 @@ def write_status_to_fetched_csv(
             fetched_path = run_dir(context["run_id"]) / "fetched.csv"
             with fetched_path.open("w", encoding="utf-8") as csv_file:
                 writer = csv.writer(csv_file)
-                writer.writerow((*records["Record ID"], "Image Path"))
 
                 status_col = records["Record ID"].index("Status")
+                records["Record ID"].insert(status_col + 1, "Status Description")
+
+                writer.writerow((*records["Record ID"], "Image Path"))
 
                 for status in statuses:
-                    record = [*records[status[0]], *status[2:]]
+                    record = [*records[status[0]], *status[3:]]
                     record[status_col] = status[1]
+                    record.insert(status_col + 1, status[2])
                     writer.writerow(record)
 
         processed = read_csv_to_process()
@@ -308,6 +312,7 @@ def transform_results(
                     "Image Name": Path(record["Image Path"]).name,
                     "Collection name": record["Collection name"],
                     "Status": record["Status"],
+                    "Status description": record.get("Status description", ""),
                     "520__a-1": record.get("Description", ""),
                     "5880_a": f"Image description generated by AI ({ENV.get('AWS_MODEL_LABEL')})"
                               " and reviewed on [MM/YYYY].",
@@ -336,8 +341,8 @@ def write_output_csv(processed_dicts: list[list[dict[str, str]]]) -> None:
                 writer.writeheader()
                 writer.writerows(all_results)
 
-        prompt = get_prompt()
         batches = read_and_batch_csv()
+        prompt = get_prompt()
         batch_results = invoke_llm_on_batch_with_prompt.partial(prompt=prompt).expand(
             batch=batches
         )

mokelumne/util/image_describer.py

@@ -43,7 +43,8 @@ def describe(self, record: dict[str, str]) -> dict[str, str]:
                 len(encoded),
                 record_meta,
             )
-            record["Status"] = "failure: file size exceeds limit"
+            record["Status"] = "failure"
+            record["Status description"] = "file size exceeds limit"
             return record
 
         image_msg = HumanMessage(
@@ -55,13 +56,15 @@ def describe(self, record: dict[str, str]) -> dict[str, str]:
                 [self.sys_msg, image_msg], config={"callbacks": [self.langfuse_handler]}
             )
         except ClientError as exc:
-            record["Status"] = f"failure: {exc.response['Error']['Message']}"
+            record["Status"] = "failure"
+            record["Status description"] = exc.response["Error"]["Message"]
             return record
 
         if hasattr(result, "content"):
             record["Status"] = "success"
             record["Description"] = str(result.content)
         else:
-            record["Status"] = "failure: no content in response"
+            record["Status"] = "failure"
+            record["Status description"] = "no content in response"
 
         return record

test/unit/test_image_describer.py

@@ -78,8 +78,7 @@ def test_client_error(self):
         describer = image_describer.ImageDescriber(model, TEST_PROMPT)
         result = describer.describe(NORMAL_RECORD_FIXTURE)
         assert "failure" in result["Status"]
-        # NOTE: When we separate the Status column, this will need to change.
-        assert err in result["Status"]
+        assert err in result["Status description"]
 
     def test_size_error(self):
         """Test case where the record's image is too large."""