Add cancel to docker executor (#233)

* Add cancel to docker executor

Signed-off-by: Hemil Desai <hemild@nvidia.com>

* fix

Signed-off-by: Hemil Desai <hemild@nvidia.com>

---------

Signed-off-by: Hemil Desai <hemild@nvidia.com>

Author: hemildesai

nemo_run/run/torchx_backend/schedulers/docker.py

@@ -254,6 +254,13 @@ def local_logs(container: DockerContainer):
         else:
             return logs
 
+    def _cancel_existing(self, app_id: str) -> None:
+        req = DockerJobRequest.load(app_id=app_id)
+        if not req:
+            return None
+        for container in req.containers:
+            container.delete(client=self._docker_client, id=req.id)
+
     def close(self) -> None:
         # terminate all apps
         for req in self._scheduled_reqs: