feat: add run_metadata column to jobs table

Author: ttngu207

datajoint/jobs.py

@@ -43,7 +43,7 @@ def __init__(self, conn, database):
         connection_id = 0  : bigint unsigned      # connection_id()
         timestamp  :timestamp   # timestamp of the job status change or scheduled time
         run_duration=null  : float  # run duration in seconds
-        run_version=""  : varchar(255) # some string representation of the code/env version of a run (e.g. git commit hash)
+        run_metadata=null  :json  # metadata about the run (e.g. code version, environment info)
         index(table_name, status)
         index(status)
         index(timestamp)  # for ordering jobs
@@ -185,14 +185,15 @@ def ignore(self, table_name, key, message=""):
 
         return True
 
-    def complete(self, table_name, key, run_duration=None, run_version=""):
+    def complete(self, table_name, key, run_duration=None, run_metadata=None):
         """
         Log a completed job.  When a job is completed, its reservation entry is deleted.
 
-        :param table_name: `database`.`table_name`
-        :param key: the dict of the job's primary key
-        :param run_duration: duration in second of the job run
-        :param run_version: some string representation of the code/env version of a run (e.g. git commit hash)
+        Args:
+            table_name: `database`.`table_name`
+            key: the dict of the job's primary key
+            run_duration: duration in second of the job run
+            run_metadata: dict containing metadata about the run (e.g. code version, environment info)
         """
         job_key = dict(table_name=table_name, key_hash=key_hash(key))
         if self & job_key:
@@ -212,24 +213,25 @@ def complete(self, table_name, key, run_duration=None, run_version=""):
                     user=self._user,
                     key=_jsonify(key),
                     run_duration=run_duration,
-                    run_version=run_version,
+                    run_metadata=_jsonify(run_metadata) if run_metadata else None,
                     timestamp=datetime.datetime.utcnow(),
                 ),
                 replace=True,
                 ignore_extra_fields=True,
             )
 
-    def error(self, table_name, key, error_message, error_stack=None, run_duration=None, run_version=""):
+    def error(self, table_name, key, error_message, error_stack=None, run_duration=None, run_metadata=None):
         """
         Log an error message.  The job reservation is replaced with an error entry.
         if an error occurs, leave an entry describing the problem
 
-        :param table_name: `database`.`table_name`
-        :param key: the dict of the job's primary key
-        :param error_message: string error message
-        :param error_stack: stack trace
-        :param run_duration: duration in second of the job run
-        :param run_version: some string representation of the code/env version of a run (e.g. git commit hash)
+        Args:
+            table_name: `database`.`table_name`
+            key: the dict of the job's primary key
+            error_message: string error message
+            error_stack: stack trace
+            run_duration: duration in second of the job run
+            run_metadata: dict containing metadata about the run (e.g. code version, environment info)
         """
         if len(error_message) > ERROR_MESSAGE_LENGTH:
             error_message = (
@@ -250,7 +252,7 @@ def error(self, table_name, key, error_message, error_stack=None, run_duration=N
                     error_message=error_message,
                     error_stack=error_stack,
                     run_duration=run_duration,
-                    run_version=run_version,
+                    run_metadata=_jsonify(run_metadata) if run_metadata else None,
                     timestamp=datetime.datetime.utcnow(),
                 ),
                 replace=True,

docs/jobs_orchestration.md

@@ -23,7 +23,7 @@ The jobs table (`~jobs`) is a system table that tracks the state and execution o
 - `connection_id`: Database connection ID
 - `timestamp`: When the job status was last changed
 - `run_duration`: How long the job took to execute (in seconds)
-- `run_version`: Version information of the code/environment
+- `run_metadata`: JSON structure containing metadata about the run (e.g. code version, environment info, system state)
 
 ## Job Scheduling Process