[4.4] Backport tests for new timeout config options [ADR 006] (#477)

Backport of #474
and #404

[ADR 006](https://github.com/neo-technology/drivers-adr/blob/main/adrs/006-acquisition-timeouts.md)

* Update and extend tests for `connection_acquisition_timeout` setting.

Tests did not encompass routing drivers

* Add new timeout driver config options

 * `sessionConnectionTimeoutMs`
 * `updateRoutingTableTimeoutMs`

* Add new feature flag
 * `API_CONNECTION_ACQUISITION_TIMEOUT = Feature:API:ConnectionAcquisitionTimeout"`

Author: robsdedude

nutkit/backend/backend.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 import inspect
 import json
 import os
@@ -59,6 +60,7 @@ def __init__(self, address, port):
         self._encoder = Encoder()
         self._reader = self._socket.makefile(mode="r", encoding="utf-8")
         self._writer = self._socket.makefile(mode="w", encoding="utf-8")
+        self.default_timeout = DEFAULT_TIMEOUT
 
     def close(self):
         self._reader.close()
@@ -79,7 +81,9 @@ def send(self, req, hooks=None):
         self._writer.write("#request end\n")
         self._writer.flush()
 
-    def receive(self, timeout=DEFAULT_TIMEOUT, hooks=None):
+    def receive(self, timeout=None, hooks=None):
+        if timeout is None:
+            timeout = self.default_timeout
         self._socket.settimeout(timeout)
         response = ""
         in_response = False
@@ -128,6 +132,14 @@ def receive(self, timeout=DEFAULT_TIMEOUT, hooks=None):
                     elif DEBUG_MESSAGES:
                         print("[BACKEND]: %s" % line)
 
-    def send_and_receive(self, req, timeout=DEFAULT_TIMEOUT, hooks=None):
+    def send_and_receive(self, req, timeout=None, hooks=None):
         self.send(req, hooks=hooks)
         return self.receive(timeout, hooks=hooks)
+
+
+@contextmanager
+def backend_timeout_adjustment(backend, timeout):
+    old_timeout = backend.default_timeout
+    backend.default_timeout = timeout
+    yield
+    backend.default_timeout = old_timeout

nutkit/frontend/driver.py

@@ -6,7 +6,8 @@ class Driver:
     def __init__(self, backend, uri, auth_token, user_agent=None,
                  resolver_fn=None, domain_name_resolver_fn=None,
                  connection_timeout_ms=None, fetch_size=None,
-                 max_tx_retry_time_ms=None, encrypted=None,
+                 max_tx_retry_time_ms=None, session_connection_timeout_ms=None,
+                 update_routing_table_timeout_ms=None, encrypted=None,
                  trusted_certificates=None, liveness_check_timeout_ms=None,
                  max_connection_pool_size=None,
                  connection_acquisition_timeout_ms=None):
@@ -18,6 +19,8 @@ def __init__(self, backend, uri, auth_token, user_agent=None,
             resolverRegistered=resolver_fn is not None,
             domainNameResolverRegistered=domain_name_resolver_fn is not None,
             connectionTimeoutMs=connection_timeout_ms,
+            sessionConnectionTimeoutMs=session_connection_timeout_ms,
+            updateRoutingTableTimeoutMs=update_routing_table_timeout_ms,
             fetchSize=fetch_size, maxTxRetryTimeMs=max_tx_retry_time_ms,
             encrypted=encrypted, trustedCertificates=trusted_certificates,
             liveness_check_timeout_ms=liveness_check_timeout_ms,

nutkit/protocol/feature.py

@@ -4,9 +4,18 @@
 
 class Feature(Enum):
     # === FUNCTIONAL FEATURES ===
+    # The driver offers a configuration option to limit time it spends at most,
+    # trying to acquire a connection from the pool.
+    # The connection acquisition timeout must account for the whole acquisition
+    # execution time, whether a new connection is created, an idle connection
+    # is picked up instead or we need to wait until the full pool depletes.
+    API_CONNECTION_ACQUISITION_TIMEOUT = \
+        "Feature:API:ConnectionAcquisitionTimeout"
     # The driver offers a method for driver objects to report if they were
     # configured with a or without encryption.
     API_DRIVER_IS_ENCRYPTED = "Feature:API:Driver.IsEncrypted"
+    # The driver supports connection liveness check.
+    API_LIVENESS_CHECK = "Feature:API:Liveness.Check"
     # The driver offers a method for the result to return all records as a list
     # or array. This method should exhaust the result.
     API_RESULT_LIST = "Feature:API:Result.List"
@@ -18,8 +27,13 @@ class Feature(Enum):
     # This methods asserts that exactly one record in left in the result
     # stream, else it will raise an exception.
     API_RESULT_SINGLE = "Feature:API:Result.Single"
-    # The driver supports connection liveness check.
-    API_LIVENESS_CHECK = "Feature:API:Liveness.Check"
+    # The driver offers a configuration option to limit time it spends at most,
+    # trying to acquire a usable read/write connection for any session.
+    # The connection acquisition timeout must account for the whole acquisition
+    # execution time, whether a new connection is created, an idle connection
+    # is picked up instead, we need to wait until the full pool depletes, or
+    # a routing table must be fetched.
+    API_SESSION_CONNECTION_TIMEOUT = "Feature:API:SessionConnectionTimeout"
     # The driver implements explicit configuration options for SSL.
     #  - enable / disable SSL
     #  - verify signature against system store / custom cert / not at all
@@ -31,6 +45,9 @@ class Feature(Enum):
     API_SSL_SCHEMES = "Feature:API:SSLSchemes"
     # The driver supports sending and receiving temporal data types.
     API_TYPE_TEMPORAL = "Feature:API:Type.Temporal"
+    # The driver offers a configuration option to limit time it spends at most,
+    # trying to update the routing table whenever needed.
+    API_UPDATE_ROUTING_TABLE_TIMEOUT = "Feature:API:UpdateRoutingTableTimeout"
     # The driver supports single-sign-on (SSO) by providing a bearer auth token
     # API.
     AUTH_BEARER = "Feature:Auth:Bearer"

nutkit/protocol/requests.py

@@ -48,6 +48,7 @@ class NewDriver:
     def __init__(
         self, uri, authToken, userAgent=None, resolverRegistered=False,
         domainNameResolverRegistered=False, connectionTimeoutMs=None,
+        sessionConnectionTimeoutMs=None, updateRoutingTableTimeoutMs=None,
         fetchSize=None, maxTxRetryTimeMs=None, encrypted=None,
         trustedCertificates=None, liveness_check_timeout_ms=None,
         max_connection_pool_size=None,
@@ -62,6 +63,8 @@ def __init__(
         self.resolverRegistered = resolverRegistered
         self.domainNameResolverRegistered = domainNameResolverRegistered
         self.connectionTimeoutMs = connectionTimeoutMs
+        self.sessionConnectionTimeoutMs = sessionConnectionTimeoutMs
+        self.updateRoutingTableTimeoutMs = updateRoutingTableTimeoutMs
         self.fetchSize = fetchSize
         self.maxTxRetryTimeMs = maxTxRetryTimeMs
         self.livenessCheckTimeoutMs = liveness_check_timeout_ms

tests/stub/driver_parameters/__init__.py

@@ -0,0 +1,12 @@
+!: BOLT 4.4
+!: AUTO RESET
+!: ALLOW RESTART
+
+A: HELLO {"{}": "*"}
+*: RESET
+{+
+    C: ROUTE "*" "*" "*"
+    S: SUCCESS { "rt": { "ttl": 1000, "servers": [{"addresses": ["#HOST#:9000"], "role":"ROUTE"}, {"addresses": ["#HOST#:9010"], "role":"READ"}, {"addresses": ["#HOST#:9010"], "role":"WRITE"}]}}
+    *: RESET
++}
+?: GOODBYE

tests/stub/driver_parameters/scripts/router.script

@@ -0,0 +1,16 @@
+!: BOLT 4.4
+!: AUTO RESET
+!: ALLOW RESTART
+
+C: HELLO {"{}": "*"}
+S: <SLEEP> 2
+   <NOOP>
+   <SLEEP> 2
+   SUCCESS {"server": "Neo4j/5.0.0", "connection_id": "bolt-123456789"}
+*: RESET
+{+
+    C: ROUTE "*" "*" "*"
+    S: SUCCESS { "rt": { "ttl": 1000, "servers": [{"addresses": ["#HOST#:9000"], "role":"ROUTE"}, {"addresses": ["#HOST#:9010"], "role":"READ"}, {"addresses": ["#HOST#:9010"], "role":"WRITE"}]}}
+    *: RESET
++}
+?: GOODBYE

tests/stub/driver_parameters/scripts/router_hello_delay.script

@@ -0,0 +1,15 @@
+!: BOLT 4.4
+!: AUTO RESET
+!: ALLOW RESTART
+
+A: HELLO {"{}": "*"}
+*: RESET
+{+
+    C: ROUTE "*" "*" "*"
+    S: <SLEEP> 2
+       <NOOP>
+       <SLEEP> 2
+       SUCCESS { "rt": { "ttl": 1000, "servers": [{"addresses": ["#HOST#:9000"], "role":"ROUTE"}, {"addresses": ["#HOST#:9010"], "role":"READ"}, {"addresses": ["#HOST#:9010"], "role":"WRITE"}]}}
+    *: RESET
++}
+?: GOODBYE

tests/stub/driver_parameters/scripts/router_route_delay.script

@@ -0,0 +1,12 @@
+!: BOLT 4.4
+!: ALLOW CONCURRENT
+
+A: HELLO {"{}": "*"}
+*: RESET
+C: RUN "*" "*" "*"
+S: SUCCESS {"fields": ["n"]}
+C: PULL {"n": "*"}
+S: RECORD [1]
+   SUCCESS {"type": "r"}
+*: RESET
+?: GOODBYE

tests/stub/driver_parameters/scripts/session_run.script

@@ -0,0 +1,16 @@
+!: BOLT 4.4
+!: ALLOW CONCURRENT
+
+C: HELLO {"{}": "*"}
+S: <SLEEP> 2
+   <NOOP>
+   <SLEEP> 2
+   SUCCESS {"server": "Neo4j/5.0.0", "connection_id": "bolt-123456789"}
+*: RESET
+C: RUN "*" "*" "*"
+S: SUCCESS {"fields": ["n"]}
+C: PULL {"n": "*"}
+S: RECORD [1]
+   SUCCESS {"type": "r"}
+*: RESET
+?: GOODBYE