Fix the scheduler timeouts and errors (TraceMachina#2083)

* Fix the scheduler timeouts by removing the keepalives

* Fix the memory leak by removing the counter

Author: amankrx

nativelink-config/examples/worker_with_redis_scheduler.json5

@@ -0,0 +1,197 @@
+{
+  stores: [
+    {
+      name: "AC_MAIN_STORE",
+      filesystem: {
+        content_path: "/tmp/nativelink/data-worker-test/content_path-ac",
+        temp_path: "/tmp/nativelink/data-worker-test/tmp_path-ac",
+        eviction_policy: {
+          max_bytes: 1000000000,
+        },
+      },
+    },
+    {
+      name: "WORKER_FAST_SLOW_STORE",
+      fast_slow: {
+        fast: {
+          filesystem: {
+            content_path: "/tmp/nativelink/data-worker-test/content_path-cas",
+            temp_path: "/tmp/nativelink/data-worker-test/tmp_path-cas",
+            eviction_policy: {
+              max_bytes: 10000000000,
+            },
+          },
+        },
+        slow: {
+          noop: {},
+        },
+      },
+    },
+    {
+      name: "SCHEDULER_REDIS_STORE",
+      redis_store: {
+        addresses: [
+          "redis://127.0.0.1:6379",
+        ],
+        connection_pool_size: 10,
+        experimental_pub_sub_channel: "scheduler_key_change",
+      },
+    },
+  ],
+  schedulers: [
+    {
+      name: "MAIN_SCHEDULER",
+      simple: {
+        worker_timeout_s: 30,
+        worker_match_logging_interval_s: -1,
+        supported_platform_properties: {
+          cpu_count: "minimum",
+          memory_kb: "minimum",
+          network_kbps: "minimum",
+          disk_read_iops: "minimum",
+          disk_read_bps: "minimum",
+          disk_write_iops: "minimum",
+          disk_write_bps: "minimum",
+          shm_size: "minimum",
+          gpu_count: "minimum",
+          gpu_model: "exact",
+          cpu_vendor: "exact",
+          cpu_arch: "exact",
+          cpu_model: "exact",
+          kernel_version: "exact",
+          OSFamily: "priority",
+          "container-image": "priority",
+          "lre-rs": "priority",
+          ISA: "exact",
+        },
+        experimental_backend: {
+          redis: {
+            redis_store: "SCHEDULER_REDIS_STORE",
+          },
+        },
+      },
+    },
+  ],
+  workers: [
+    {
+      local: {
+        worker_api_endpoint: {
+          uri: "grpc://127.0.0.1:50061",
+        },
+        cas_fast_slow_store: "WORKER_FAST_SLOW_STORE",
+        upload_action_result: {
+          ac_store: "AC_MAIN_STORE",
+        },
+        work_directory: "/tmp/nativelink/work",
+        platform_properties: {
+          cpu_count: {
+            values: [
+              "14",
+            ],
+          },
+          memory_kb: {
+            values: [
+              "32000000",
+            ],
+          },
+          network_kbps: {
+            values: [
+              "100000",
+            ],
+          },
+          cpu_arch: {
+            values: [
+              "aarch64",
+            ],
+          },
+          OSFamily: {
+            values: [
+              "Darwin",
+              "",
+            ],
+          },
+          "container-image": {
+            values: [
+              "",
+            ],
+          },
+          "lre-rs": {
+            values: [
+              "",
+            ],
+          },
+          ISA: {
+            values: [
+              "aarch64",
+            ],
+          },
+        },
+      },
+    },
+  ],
+  servers: [
+    {
+      name: "public",
+      listener: {
+        http: {
+          socket_address: "0.0.0.0:50051",
+        },
+      },
+      services: {
+        cas: [
+          {
+            instance_name: "main",
+            cas_store: "WORKER_FAST_SLOW_STORE",
+          },
+        ],
+        ac: [
+          {
+            instance_name: "main",
+            ac_store: "AC_MAIN_STORE",
+          },
+        ],
+        execution: [
+          {
+            instance_name: "main",
+            cas_store: "WORKER_FAST_SLOW_STORE",
+            scheduler: "MAIN_SCHEDULER",
+          },
+        ],
+        capabilities: [
+          {
+            instance_name: "main",
+            remote_execution: {
+              scheduler: "MAIN_SCHEDULER",
+            },
+          },
+        ],
+        bytestream: [
+          {
+            instance_name: "main",
+            cas_store: "WORKER_FAST_SLOW_STORE",
+          },
+        ],
+        health: {},
+        admin: {},
+      },
+    },
+    {
+      name: "private_workers_servers",
+      listener: {
+        http: {
+          socket_address: "0.0.0.0:50061",
+        },
+      },
+      services: {
+        worker_api: {
+          scheduler: "MAIN_SCHEDULER",
+        },
+        admin: {},
+        health: {},
+      },
+    },
+  ],
+  global: {
+    max_open_files: 24576,
+  },
+}

nativelink-scheduler/BUILD.bazel

@@ -25,6 +25,7 @@ rust_library(
         "src/simple_scheduler_state_manager.rs",
         "src/store_awaited_action_db.rs",
         "src/worker.rs",
+        "src/worker_registry.rs",
         "src/worker_scheduler.rs",
     ],
     proc_macro_deps = [