Add support for gRPC TLS server authentication

Signed-off-by: Asher Pemberton <asher.pemberton@arm.com>
Reviewed-by: Asher Pemberton <asher.pemberton@arm.com> # gatekeeper
Co-authored-by: Luke Beardsmore <luke.beardsmore2@arm.com>

Author: asher-pem-arm

doc/getting_started.rst

@@ -435,6 +435,26 @@ Follow these instructions to install the systemd files on your machine(s):
 
       # usermod -a -G labgrid <user>
 
+Enabling gRPC connection security
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is encouraged to use TLS for gRPC channels in a production environment.
+
+This can be enabled on the ``labgrid-coordinator`` by adding the ``--tls``,
+``--cert`` and ``--key`` options.
+Refer to the ``labgrid-coordinator`` man page for details.
+
+When you are connecting with ``labgrid-client`` or ``labgrid-exporter`` to a
+``labgrid-coordinator`` that has TLS gRPC channels enabled you need to pass
+the ``--tls`` option. If ``--cert`` is not set, labgrid uses the host CA
+certificates to verify the coordinator certificate. Use ``--cert`` to provide
+a specific CA certificate instead.
+Refer to the ``labgrid-client`` and ``labgrid-exporter`` man pages for details.
+For ``RemotePlace`` connections from an environment config, set the
+``coordinator_tls`` option or ``LG_COORDINATOR_TLS``. If ``coordinator_cert``
+is not set, labgrid uses the host CA certificates. Set ``coordinator_cert``
+to provide a specific CA certificate instead.
+
 Using a Strategy
 ----------------
 

doc/man/client.rst

@@ -54,6 +54,11 @@ LG_COORDINATOR
 This variable can be used to set the default coordinator in the format
 ``HOST[:PORT]`` (instead of using the ``-x`` option).
 
+LG_COORDINATOR_TLS
+~~~~~~~~~~~~~~~~~~
+This variable can be set to enable TLS gRPC channels without using the
+``--tls`` option.
+
 LG_PROXY
 ~~~~~~~~
 This variable can be used to specify a SSH proxy hostname which should be used
@@ -128,6 +133,12 @@ Add all resources with the group "example-group" to the place example-place:
 
    $ labgrid-client -p example-place add-match */example-group/*/*
 
+Retrieve a list of places on a ``labgrid-coordinator`` that uses TLS gRPC channels:
+
+.. code-block:: bash
+
+    $ labgrid-client --tls [--cert PATH] places
+
 See Also
 --------
 

doc/man/coordinator.rst

@@ -25,9 +25,41 @@ OPTIONS
     display command line help
 -l ADDRESS, --listen ADDRESS
     make coordinator listen on host and port
+--tls
+    enable TLS gRPC channel
+--cert
+    path to TLS certificate (in PEM format)
+--key
+    path to TLS key (in PEM format)
 -d, --debug
     enable debug mode
 
+TLS WITH A REVERSE PROXY
+------------------------
+
+Instead of enabling TLS in ``labgrid-coordinator`` directly, a reverse proxy can
+terminate TLS and forward cleartext gRPC to the coordinator. For example, with
+``nginx``:
+
+.. code-block:: nginx
+
+   server {
+       listen 20407 ssl http2;
+       server_name labgrid.example.com;
+
+       ssl_certificate     /etc/ssl/labgrid-coordinator.crt;
+       ssl_certificate_key /etc/ssl/labgrid-coordinator.key;
+
+       location / {
+           grpc_pass grpc://127.0.0.1:20408;
+       }
+   }
+
+In this setup, start ``labgrid-coordinator`` without ``--tls`` and point
+``labgrid-client`` and ``labgrid-exporter`` at the reverse proxy using
+``--tls``.
+
+
 SEE ALSO
 --------
 

doc/man/device-config.rst

@@ -43,6 +43,15 @@ OPTIONS KEYS
   takes as parameter the coordinator ``HOST[:PORT]`` to connect to.
   Defaults to ``127.0.0.1:20408``.
 
+``coordinator_tls``
+  enables a TLS gRPC channel for ``RemotePlace`` connections.
+  Defaults to whether ``LG_COORDINATOR_TLS`` is set.
+
+``coordinator_cert``
+  path to the coordinator TLS certificate in PEM format for ``RemotePlace``
+  connections. If unset, the host CA certificates are used. Relative paths are
+  resolved relative to the configuration file.
+
 .. _labgrid-device-config-images:
 
 IMAGES

doc/man/exporter.rst

@@ -27,6 +27,10 @@ OPTIONS
     display command line help
 -x, --coordinator
     coordinator ``HOST[:PORT]`` to connect to, defaults to ``127.0.0.1:20408``
+--tls
+    enable TLS gRPC channel
+--cert
+    path to TLS certificate (in PEM format)
 -i, --isolated
     enable isolated mode (always request SSH forwards)
 -n, --name
@@ -99,6 +103,12 @@ Same as above, but with name ``myname``:
 
    $ labgrid-exporter -n myname my-config.yaml
 
+Same as above, but connecting to a ``labgrid-coordinator`` that uses TLS gRPC channels:
+
+.. code-block:: bash
+
+    $ labgrid-exporter --tls [--cert PATH] -n myname my-config.yaml
+
 SEE ALSO
 --------
 

labgrid/remote/client.py

@@ -41,6 +41,7 @@
     TAG_KEY,
     TAG_VAL,
     queue_as_aiter,
+    get_client_credentials,
 )
 from .. import Environment, Target, target_factory
 from ..exceptions import NoDriverFoundError, NoResourceFoundError, InvalidConfigError
@@ -92,6 +93,7 @@ class ClientSession:
     the coordinator."""
 
     address = attr.ib(validator=attr.validators.instance_of(str))
+    credentials = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(grpc.ChannelCredentials)))
     loop = attr.ib(validator=attr.validators.instance_of(asyncio.BaseEventLoop))
     env = attr.ib(default=None, validator=attr.validators.optional(attr.validators.instance_of(Environment)))
     role = attr.ib(default=None, validator=attr.validators.optional(attr.validators.instance_of(str)))
@@ -120,10 +122,18 @@ def __attrs_post_init__(self):
             ("grpc.http2.max_pings_without_data", 0),  # no limit
         ]
 
-        self.channel = grpc.aio.insecure_channel(
-            target=self.address,
-            options=channel_options,
-        )
+        if self.credentials:
+            self.channel = grpc.aio.secure_channel(
+                target=self.address,
+                credentials=self.credentials,
+                options=channel_options,
+            )
+        else:
+            self.channel = grpc.aio.insecure_channel(
+                target=self.address,
+                options=channel_options,
+            )
+
         self.stub = labgrid_coordinator_pb2_grpc.CoordinatorStub(self.channel)
 
         self.out_queue = asyncio.Queue()
@@ -1698,7 +1708,12 @@ def ensure_event_loop(external_loop=None):
 
 
 def start_session(
-    address: str, *, extra: Dict[str, Any] = None, debug: bool = False, loop: "asyncio.AbstractEventLoop | None" = None
+    address: str,
+    *,
+    extra: Dict[str, Any] = None,
+    credentials: grpc.ChannelCredentials = None,
+    debug: bool = False,
+    loop: "asyncio.AbstractEventLoop | None" = None,
 ):
     """
     Starts a ClientSession.
@@ -1720,7 +1735,7 @@ def start_session(
 
     address = proxymanager.get_grpc_address(address, default_port=20408)
 
-    session = ClientSession(address, loop, **extra)
+    session = ClientSession(address, credentials, loop, **extra)
     loop.run_until_complete(session.start())
     return session
 
@@ -1848,6 +1863,13 @@ def get_parser(auto_doc_mode=False) -> "argparse.ArgumentParser | AutoProgramArg
         type=str,
         help="coordinator HOST[:PORT] (default: value from env variable LG_COORDINATOR, otherwise 127.0.0.1:20408)",
     )
+    parser.add_argument(
+        "--tls",
+        action="store_true",
+        default=os.environ.get("LG_COORDINATOR_TLS") is not None,
+        help="enable TLS gRPC channel",
+    )
+    parser.add_argument("--cert", type=pathlib.PurePath, help="path to the server's TLS certificate (in PEM format)")
     parser.add_argument(
         "-c",
         "--config",
@@ -2346,7 +2368,9 @@ def main():
             logging.debug('Starting session with "%s"', coordinator_address)
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
-            session = start_session(coordinator_address, extra=extra, debug=args.debug, loop=loop)
+            session = start_session(
+                coordinator_address, extra=extra, credentials=get_client_credentials(args), debug=args.debug, loop=loop
+            )
             logging.debug("Started session")
 
             try:

labgrid/remote/common.py

@@ -5,10 +5,12 @@
 import re
 import string
 import logging
+from argparse import Namespace
 from datetime import datetime
 from fnmatch import fnmatchcase
-
+from typing import Optional
 import attr
+import grpc
 
 from .generated import labgrid_coordinator_pb2
 
@@ -58,6 +60,17 @@ def build_dict_from_map(m):
     return d
 
 
+def get_client_credentials(args: Namespace) -> Optional[grpc.ChannelCredentials]:
+    if not args.tls:
+        return None
+
+    if not args.cert:
+        return grpc.ssl_channel_credentials()
+
+    with open(args.cert, "rb") as fc:
+        return grpc.ssl_channel_credentials(fc.read())
+
+
 @attr.s(eq=False)
 class ResourceEntry:
     data = attr.ib()  # cls, params

labgrid/remote/coordinator.py

@@ -10,7 +10,8 @@
 import copy
 import random
 import signal
-
+import pathlib
+from typing import Optional
 import attr
 import grpc
 from grpc_reflection.v1alpha import reflection
@@ -1109,7 +1110,7 @@ async def GetReservations(self, request: labgrid_coordinator_pb2.GetReservations
         return labgrid_coordinator_pb2.GetReservationsResponse(reservations=reservations)
 
 
-async def serve(listen, cleanup) -> None:
+async def serve(listen, cleanup, server_credentials=None) -> None:
     asyncio.current_task().set_name("coordinator-serve")
     # It seems since https://github.com/grpc/grpc/pull/34647, the
     # ping_timeout_ms default of 60 seconds overrides keepalive_timeout_ms,
@@ -1146,7 +1147,11 @@ async def serve(listen, cleanup) -> None:
     except ImportError:
         logging.info("Module grpcio-channelz not available")
 
-    bound = server.add_insecure_port(listen)
+    if server_credentials:
+        bound = server.add_secure_port(listen, server_credentials)
+    else:
+        bound = server.add_insecure_port(listen)
+
     logging.debug("Starting server")
     await server.start()
 
@@ -1175,6 +1180,18 @@ def callback():
     await server.wait_for_termination()
 
 
+def get_server_credentials(args: argparse.Namespace) -> Optional[grpc.ServerCredentials]:
+    if not args.tls:
+        return None
+
+    if not args.cert or not args.key:
+        raise RuntimeError("--cert and --key must be provided when --tls is provided")
+
+    with open(args.key, "rb") as fk:
+        with open(args.cert, "rb") as fc:
+            return grpc.ssl_server_credentials([(fk.read(), fc.read())])
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -1185,6 +1202,9 @@ def main():
         default="[::]:20408",
         help="coordinator listening host and port",
     )
+    parser.add_argument("--tls", action="store_true", default=False, help="enable TLS gRPC channel")
+    parser.add_argument("--cert", type=pathlib.PurePath, help="path to TLS certificate (in PEM format)")
+    parser.add_argument("--key", type=pathlib.PurePath, help="path to TLS key (in PEM format)")
     parser.add_argument("-d", "--debug", action="store_true", default=False, help="enable debug mode")
     parser.add_argument("--pystuck", action="store_true", help="enable pystuck")
     parser.add_argument(
@@ -1214,7 +1234,8 @@ def main():
     cleanup = []
     loop.set_debug(True)
     try:
-        loop.run_until_complete(serve(args.listen, cleanup))
+        server_credentials = get_server_credentials(args)
+        loop.run_until_complete(serve(args.listen, cleanup, server_credentials))
     finally:
         if cleanup:
             loop.run_until_complete(*cleanup)

labgrid/remote/exporter.py

@@ -16,12 +16,13 @@
 from pathlib import Path
 from typing import Dict, Type
 from socket import gethostname, getfqdn
+import pathlib
 
 import attr
 import grpc
 
 from .config import ResourceConfig
-from .common import ResourceEntry, queue_as_aiter
+from .common import ResourceEntry, get_client_credentials, queue_as_aiter
 from .generated import labgrid_coordinator_pb2, labgrid_coordinator_pb2_grpc
 from ..util import get_free_port, labgrid_version
 
@@ -831,10 +832,17 @@ def __init__(self, config) -> None:
         if urlsplit(f"//{config['coordinator']}").port is None:
             config["coordinator"] += ":20408"
 
-        self.channel = grpc.aio.insecure_channel(
-            target=config["coordinator"],
-            options=channel_options,
-        )
+        if config["credentials"]:
+            self.channel = grpc.aio.secure_channel(
+                target=config["coordinator"],
+                credentials=config["credentials"],
+                options=channel_options,
+            )
+        else:
+            self.channel = grpc.aio.insecure_channel(
+                target=config["coordinator"],
+                options=channel_options,
+            )
         self.stub = labgrid_coordinator_pb2_grpc.CoordinatorStub(self.channel)
         self.out_queue = asyncio.Queue()
         self.pump_task = None
@@ -1081,6 +1089,8 @@ def main():
         default=os.environ.get("LG_COORDINATOR", "127.0.0.1:20408"),
         help="coordinator host and port",
     )
+    parser.add_argument("--tls", action="store_true", default=False, help="enable TLS gRPC channel")
+    parser.add_argument("--cert", type=pathlib.PurePath, help="path to TLS certificate (in PEM format)")
     parser.add_argument(
         "-n",
         "--name",
@@ -1122,6 +1132,7 @@ def main():
         "hostname": args.hostname or (getfqdn() if args.fqdn else gethostname()),
         "resources": args.resources,
         "coordinator": args.coordinator,
+        "credentials": get_client_credentials(args),
         "isolated": args.isolated,
     }