feat: add endpoint option

This implements ADR-119[1], which specifies the client connection
options to update requests to the endpoints implemented as part of
ADR-042[2].

The endpoint may be one of the following:

* a routing policy name (such as main)
* a nonprod routing policy name (such as nonprod:sandbox)
* a FQDN such as foo.example.com

The endpoint option is not valid with any of environment, restHost or
realtimeHost, but we still intend to support the legacy options.

If the client has been configured to use any of these legacy options,
then they should continue to work in the same way, using the same
primary and fallback hostnames.

If the client has not been explicitly configured, then the hostnames
will change to the new ably.net domain when the package is upgraded.

[1] https://ably.atlassian.net/wiki/spaces/ENG/pages/3428810778/ADR-119+ClientOptions+for+new+DNS+structure
[2] https://ably.atlassian.net/wiki/spaces/ENG/pages/1791754276/ADR-042+DNS+Restructure

Author: surminus

ably/realtime/realtime.py

@@ -48,10 +48,14 @@ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEve
             You can set this to false and explicitly connect to Ably using the
             connect() method. The default is true.
         **kwargs: client options
+            endpoint: str
+                Endpoint specifies either a routing policy name or fully qualified domain name to connect to Ably.
             realtime_host: str
+                Deprecated: this property is deprecated and will be removed in a future version.
                 Enables a non-default Ably host to be specified for realtime connections.
                 For development environments only. The default value is realtime.ably.io.
             environment: str
+                Deprecated: this property is deprecated and will be removed in a future version.
                 Enables a custom environment to be used with the Ably service. Defaults to `production`
             realtime_request_timeout: float
                 Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime

ably/rest/rest.py

@@ -32,8 +32,14 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None,
 
           **Optional Parameters**
           - `client_id`: Undocumented
-          - `rest_host`: The host to connect to. Defaults to rest.ably.io
-          - `environment`: The environment to use. Defaults to 'production'
+          - `endpoint`: Endpoint specifies either a routing policy name or
+            fully qualified domain name to connect to Ably.
+          - `rest_host`: Deprecated: this property is deprecated and will
+            be removed in a future version. The host to connect to.
+            Defaults to rest.ably.io
+          - `environment`: Deprecated: this property is deprecated and
+            will be removed in a future version. The environment to use.
+            Defaults to 'production'
           - `port`: The port to connect to. Defaults to 80
           - `tls_port`: The tls_port to connect to. Defaults to 443
           - `tls`: Specifies whether the client should use TLS. Defaults

ably/transport/defaults.py

@@ -1,17 +1,8 @@
 class Defaults:
     protocol_version = "2"
-    fallback_hosts = [
-        "a.ably-realtime.com",
-        "b.ably-realtime.com",
-        "c.ably-realtime.com",
-        "d.ably-realtime.com",
-        "e.ably-realtime.com",
-    ]
-
-    rest_host = "rest.ably.io"
-    realtime_host = "realtime.ably.io"  # RTN2
+
     connectivity_check_url = "https://internet-up.ably-realtime.com/is-the-internet-up.txt"
-    environment = 'production'
+    endpoint = 'main'
 
     port = 80
     tls_port = 443
@@ -53,11 +44,34 @@ def get_scheme(options):
             return "http"
 
     @staticmethod
-    def get_environment_fallback_hosts(environment):
+    def get_hostname(endpoint):
+        if "." in endpoint or "::" in endpoint or "localhost" in endpoint:
+            return endpoint
+
+        if endpoint.startswith("nonprod:"):
+            return endpoint[len("nonprod:"):] + ".realtime.ably-nonprod.net"
+
+        return endpoint + ".realtime.ably.net"
+
+    @staticmethod
+    def get_fallback_hosts(endpoint="main"):
+        if "." in endpoint or "::" in endpoint or "localhost" in endpoint:
+            return []
+
+        if endpoint.startswith("nonprod:"):
+            root = endpoint.replace("nonprod:", "")
+            return [
+                root + ".a.fallback.ably-realtime-nonprod.com",
+                root + ".b.fallback.ably-realtime-nonprod.com",
+                root + ".c.fallback.ably-realtime-nonprod.com",
+                root + ".d.fallback.ably-realtime-nonprod.com",
+                root + ".e.fallback.ably-realtime-nonprod.com",
+            ]
+
         return [
-            environment + "-a-fallback.ably-realtime.com",
-            environment + "-b-fallback.ably-realtime.com",
-            environment + "-c-fallback.ably-realtime.com",
-            environment + "-d-fallback.ably-realtime.com",
-            environment + "-e-fallback.ably-realtime.com",
+            endpoint + ".a.fallback.ably-realtime.com",
+            endpoint + ".b.fallback.ably-realtime.com",
+            endpoint + ".c.fallback.ably-realtime.com",
+            endpoint + ".d.fallback.ably-realtime.com",
+            endpoint + ".e.fallback.ably-realtime.com",
         ]

ably/types/options.py

@@ -26,16 +26,21 @@ def decode(self, delta: bytes, base: bytes) -> bytes:
 
 class Options(AuthOptions):
     def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0,
-                 tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, environment=None,
-                 http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None,
-                 http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None,
-                 fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None,
-                 loop=None, auto_connect=True, suspended_retry_timeout=None, connectivity_check_url=None,
+                 tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None,
+                 environment=None, http_open_timeout=None, http_request_timeout=None,
+                 realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None,
+                 fallback_hosts=None, fallback_retry_timeout=None, disconnected_retry_timeout=None,
+                 idempotent_rest_publishing=None, loop=None, auto_connect=True,
+                 suspended_retry_timeout=None, connectivity_check_url=None,
                  channel_retry_timeout=Defaults.channel_retry_timeout, add_request_ids=False,
                  vcdiff_decoder: VCDiffDecoder = None, transport_params=None, **kwargs):
 
         super().__init__(**kwargs)
 
+        if endpoint is not None:
+            if environment is not None or rest_host is not None or realtime_host is not None:
+                raise ValueError('endpoint is incompatible with any of environment, rest_host or realtime_host')
+
         # TODO check these defaults
         if fallback_retry_timeout is None:
             fallback_retry_timeout = Defaults.fallback_retry_timeout
@@ -64,8 +69,11 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti
             from ably import api_version
             idempotent_rest_publishing = api_version >= '1.2'
 
-        if environment is None:
-            environment = Defaults.environment
+        if environment is not None and endpoint is None:
+            endpoint = environment
+
+        if endpoint is None:
+            endpoint = Defaults.endpoint
 
         self.__client_id = client_id
         self.__log_level = log_level
@@ -77,7 +85,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti
         self.__use_binary_protocol = use_binary_protocol
         self.__queue_messages = queue_messages
         self.__recover = recover
-        self.__environment = environment
+        self.__endpoint = endpoint
         self.__http_open_timeout = http_open_timeout
         self.__http_request_timeout = http_request_timeout
         self.__realtime_request_timeout = realtime_request_timeout
@@ -183,8 +191,8 @@ def recover(self, value):
         self.__recover = value
 
     @property
-    def environment(self):
-        return self.__environment
+    def endpoint(self):
+        return self.__endpoint
 
     @property
     def http_open_timeout(self):
@@ -296,27 +304,19 @@ def __get_rest_hosts(self):
         # Defaults
         host = self.rest_host
         if host is None:
-            host = Defaults.rest_host
-
-        environment = self.environment
+            host = Defaults.get_hostname(self.endpoint)
 
         http_max_retry_count = self.http_max_retry_count
         if http_max_retry_count is None:
             http_max_retry_count = Defaults.http_max_retry_count
 
-        # Prepend environment
-        if environment != 'production':
-            host = f'{environment}-{host}'
-
         # Fallback hosts
         fallback_hosts = self.fallback_hosts
         if fallback_hosts is None:
-            if host == Defaults.rest_host:
-                fallback_hosts = Defaults.fallback_hosts
-            elif environment != 'production':
-                fallback_hosts = Defaults.get_environment_fallback_hosts(environment)
-            else:
+            if self.rest_host is not None:
                 fallback_hosts = []
+            else:
+                fallback_hosts = Defaults.get_fallback_hosts(self.endpoint)
 
         # Shuffle
         fallback_hosts = list(fallback_hosts)
@@ -332,11 +332,8 @@ def __get_realtime_hosts(self):
         if self.realtime_host is not None:
             host = self.realtime_host
             return [host]
-        elif self.environment != "production":
-            host = f'{self.environment}-{Defaults.realtime_host}'
-        else:
-            host = Defaults.realtime_host
 
+        host = Defaults.get_hostname(self.endpoint)
         return [host] + self.__fallback_hosts
 
     def get_rest_hosts(self):

test/ably/rest/restinit_test.py

@@ -73,15 +73,15 @@ def test_rest_host_and_environment(self):
         ably = AblyRest(token='foo', rest_host="some.other.host")
         assert "some.other.host" == ably.options.rest_host, "Unexpected host mismatch"
 
-        # environment: production
-        ably = AblyRest(token='foo', environment="production")
+        # environment: main
+        ably = AblyRest(token='foo', environment="main")
         host = ably.options.get_rest_host()
-        assert "rest.ably.io" == host, f"Unexpected host mismatch {host}"
+        assert "main.realtime.ably.net" == host, f"Unexpected host mismatch {host}"
 
         # environment: other
-        ably = AblyRest(token='foo', environment="sandbox")
+        ably = AblyRest(token='foo', environment="nonprod:sandbox")
         host = ably.options.get_rest_host()
-        assert "sandbox-rest.ably.io" == host, f"Unexpected host mismatch {host}"
+        assert "sandbox.realtime.ably-nonprod.net" == host, f"Unexpected host mismatch {host}"
 
         # both, as per #TO3k2
         with pytest.raises(ValueError):
@@ -103,13 +103,13 @@ def test_fallback_hosts(self):
             assert sorted(aux) == sorted(ably.options.get_fallback_rest_hosts())
 
         # Specify environment (RSC15g2)
-        ably = AblyRest(token='foo', environment='sandbox', http_max_retry_count=10)
-        assert sorted(Defaults.get_environment_fallback_hosts('sandbox')) == sorted(
+        ably = AblyRest(token='foo', environment='nonprod:sandbox', http_max_retry_count=10)
+        assert sorted(Defaults.get_fallback_hosts('nonprod:sandbox')) == sorted(
             ably.options.get_fallback_rest_hosts())
 
         # Fallback hosts and environment not specified (RSC15g3)
         ably = AblyRest(token='foo', http_max_retry_count=10)
-        assert sorted(Defaults.fallback_hosts) == sorted(ably.options.get_fallback_rest_hosts())
+        assert sorted(Defaults.get_fallback_hosts()) == sorted(ably.options.get_fallback_rest_hosts())
 
         # RSC15f
         ably = AblyRest(token='foo')
@@ -182,13 +182,17 @@ async def test_query_time_param(self):
     @dont_vary_protocol
     def test_requests_over_https_production(self):
         ably = AblyRest(token='token')
-        assert 'https://rest.ably.io' == f'{ably.http.preferred_scheme}://{ably.http.preferred_host}'
+        assert 'https://main.realtime.ably.net' == f'{
+            ably.http.preferred_scheme}://{ ably.http.preferred_host
+        }'
         assert ably.http.preferred_port == 443
 
     @dont_vary_protocol
     def test_requests_over_http_production(self):
         ably = AblyRest(token='token', tls=False)
-        assert 'http://rest.ably.io' == f'{ably.http.preferred_scheme}://{ably.http.preferred_host}'
+        assert 'http://main.realtime.ably.net' == f'{
+            ably.http.preferred_scheme}://{ ably.http.preferred_host
+        }'
         assert ably.http.preferred_port == 80
 
     @dont_vary_protocol
@@ -211,7 +215,7 @@ async def test_environment(self):
             except AblyException:
                 pass
             request = get_mock.call_args_list[0][0][0]
-            assert request.url == 'https://custom-rest.ably.io:443/time'
+            assert request.url == 'https://custom.realtime.ably.net:443/time'
 
         await ably.close()
 

test/ably/rest/restpaginatedresult_test.py

@@ -32,7 +32,7 @@ async def setup(self):
         self.ably = await TestApp.get_ably_rest(use_binary_protocol=False)
         # Mocked responses
         # without specific headers
-        self.mocked_api = respx.mock(base_url='http://rest.ably.io')
+        self.mocked_api = respx.mock(base_url='http://main.realtime.ably.net')
         self.ch1_route = self.mocked_api.get('/channels/channel_name/ch1')
         self.ch1_route.return_value = Response(
             headers={'content-type': 'application/json'},
@@ -45,8 +45,8 @@ async def setup(self):
             headers={
                 'content-type': 'application/json',
                 'link':
-                    '<http://rest.ably.io/channels/channel_name/ch2?page=1>; rel="first",'
-                    ' <http://rest.ably.io/channels/channel_name/ch2?page=2>; rel="next"'
+                    '<http://main.realtime.ably.net/channels/channel_name/ch2?page=1>; rel="first",'
+                    ' <http://main.realtime.ably.net/channels/channel_name/ch2?page=2>; rel="next"'
             },
             body='[{"id": 0}, {"id": 1}]',
             status=200
@@ -56,11 +56,11 @@ async def setup(self):
 
         self.paginated_result = await PaginatedResult.paginated_query(
             self.ably.http,
-            url='http://rest.ably.io/channels/channel_name/ch1',
+            url='http://main.realtime.ably.net/channels/channel_name/ch1',
             response_processor=lambda response: response.to_native())
         self.paginated_result_with_headers = await PaginatedResult.paginated_query(
             self.ably.http,
-            url='http://rest.ably.io/channels/channel_name/ch2',
+            url='http://main.realtime.ably.net/channels/channel_name/ch2',
             response_processor=lambda response: response.to_native())
         yield
         self.mocked_api.stop()

test/ably/rest/restrequest_test.py

@@ -100,8 +100,8 @@ async def test_timeout(self):
             await ably.request('GET', '/time', version=Defaults.protocol_version)
         await ably.close()
 
-        default_endpoint = 'https://sandbox-rest.ably.io/time'
-        fallback_host = 'sandbox-a-fallback.ably-realtime.com'
+        default_endpoint = 'https://sandbox.realtime.ably-nonprod.net/time'
+        fallback_host = 'sandbox.a.fallback.ably-realtime-nonprod.com'
         fallback_endpoint = f'https://{fallback_host}/time'
         ably = await TestApp.get_ably_rest(fallback_hosts=[fallback_host])
         with respx.mock:

test/ably/testapp.py

@@ -14,15 +14,15 @@
     app_spec_local = json.loads(f.read())
 
 tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true"
-rest_host = os.environ.get('ABLY_REST_HOST', 'sandbox-rest.ably.io')
-realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox-realtime.ably.io')
+rest_host = os.environ.get('ABLY_REST_HOST', 'sandbox.realtime.ably-nonprod.net')
+realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox.realtime.ably-nonprod.net')
 
-environment = os.environ.get('ABLY_ENV', 'sandbox')
+environment = os.environ.get('ABLY_ENDPOINT', 'nonprod:sandbox')
 
 port = 80
 tls_port = 443
 
-if rest_host and not rest_host.endswith("rest.ably.io"):
+if rest_host and not rest_host.endswith("realtime.ably-nonprod.net"):
     tls = tls and rest_host != "localhost"
     port = 8080
     tls_port = 8081

test/unit/options_test.py

@@ -0,0 +1,61 @@
+import pytest
+
+from ably.types.options import Options
+
+
+def test_options_should_fail_early_with_incompatible_client_options():
+    with pytest.raises(ValueError):
+        Options(endpoint="foo", environment="foo")
+
+    with pytest.raises(ValueError):
+        Options(endpoint="foo", rest_host="foo")
+
+    with pytest.raises(ValueError):
+        Options(endpoint="foo", realtime_host="foo")
+
+
+# REC1a
+def test_options_should_return_the_default_hostnames():
+    opts = Options()
+    assert opts.get_realtime_host() == "main.realtime.ably.net"
+    assert "main.a.fallback.ably-realtime.com" in opts.get_fallback_realtime_hosts()
+
+
+# REC1b4
+def test_options_should_return_the_correct_routing_policy_hostnames():
+    opts = Options(endpoint="foo")
+    assert opts.get_realtime_host() == "foo.realtime.ably.net"
+    assert "foo.a.fallback.ably-realtime.com" in opts.get_fallback_realtime_hosts()
+
+
+# REC1b3
+def test_options_should_return_the_correct_nonprod_routing_policy_hostnames():
+    opts = Options(endpoint="nonprod:foo")
+    assert opts.get_realtime_host() == "foo.realtime.ably-nonprod.net"
+    assert "foo.a.fallback.ably-realtime-nonprod.com" in opts.get_fallback_realtime_hosts()
+
+
+# REC1b2
+def test_options_should_return_the_correct_fqdn_hostnames():
+    opts = Options(endpoint="foo.com")
+    assert opts.get_realtime_host() == "foo.com"
+    assert not opts.get_fallback_realtime_hosts()
+
+
+# REC1b2
+def test_options_should_return_an_ipv4_address():
+    opts = Options(endpoint="127.0.0.1")
+    assert opts.get_realtime_host() == "127.0.0.1"
+    assert not opts.get_fallback_realtime_hosts()
+
+
+# REC1b2
+def test_options_should_return_an_ipv6_address():
+    opts = Options(endpoint="::1")
+    assert opts.get_realtime_host() == "::1"
+
+
+# REC1b2
+def test_options_should_return_localhost():
+    opts = Options(endpoint="localhost")
+    assert opts.get_realtime_host() == "localhost"