asyncio CancelledError and TimeoutError when reconnecting to host in faillover mode #1309
Description
Struggling with my "feature", caused, probably, by my "straight" hands, for around 3 days already.
The infra setup - PG cluster with 2 nodes, managed by patroni. Node1 - Primary, Node2 - Sync Standby (replica). The flow is simple. Working on Node1, until it crashes. Than Node2 becomes Primary and we switch there. At this time Node1 is rebooting and becomes Replica. We work in Node2, until it crashes, than Node1 becomes Primary, Node2 rebooting and becomes Replica again.
All works fine, until we reach the last step of the flow, when service trying to switch to Node1 again. This time all pool.acquire() attempts fails with asyncio.CanceledError and Timeout.
Okie, (said I to myself), lets decorate it with error handler and recreate the pool after a couple of failed execute() attempts. But it didn't help. Even with pool.close() before pool recreation.
I've crashed into multiple issues across the sqlalchemy and asyncpg repos, which contains semirelated cases. Looks like the root problem is somewhere in server connection lock. Also, my current app realisation somehow unfreezes and lets me to recreate the pool after a couple of minutes of waiting and non-requesting db, so "connection lock" looks realistic for me.
But I'm definitely stuck in understanding, what can I do in app, to prevent this behaviour.
Want to note, that I try query execution only after server primary node is fully alive.
Here is the "absolute gem" of fever dream coding (sorry for that):
import asyncio
import functools
import logging
from uuid import uuid4
import asyncpg
from app import CFG
class AsyncPGClient:
def __init__(self):
self._dsn = CFG.database.dsn
self._schema = CFG.database.schema
self._table = CFG.database.table
self._user = CFG.database.user
self._password = CFG.database.password
self._conn_timeout = 10
self._command_timeout = 10
self._target_session_attrs = 'primary'
self._pool: asyncpg.Pool | None = None
@staticmethod
def retry(max_retries: int = 5, retry_delay: int = 3, try_to_reconnect: bool = True, raise_exc: bool = True):
def decorator(func):
@functools.wraps(func)
async def wrapper(self, *args, **kwargs):
_try_to_reconnect = try_to_reconnect
attempt = 0
while attempt <= max_retries:
try:
return await func(self, *args, **kwargs)
except Exception as error:
attempt += 1
logging.error(f'Failed to {self.__class__.__name__}.{func.__name__}().', exc_info=error)
if attempt > max_retries:
if func.__name__ is not self.connect.__name__:
_try_to_reconnect = False
attempt = 1
await self.connect()
elif raise_exc:
raise error
logging.warning(f'Retrying {self.__class__.__name__}.{func.__name__}() in {retry_delay}s. '
f'Attempt: {attempt}/{max_retries}.')
await asyncio.sleep(retry_delay)
return wrapper
return decorator
@retry(3, 3, False)
async def connect(self):
await self.close()
logging.info(f'Creating connection pool for "{self._dsn}".')
self._pool = await asyncpg.create_pool(dsn=self._dsn,
target_session_attrs='primary',
server_settings={'search_path': self._schema},
user=self._user, password=self._password,
timeout=self._conn_timeout, command_timeout=self._command_timeout,
min_size=1, max_size=10)
logging.info(f'Successfully created connection pool for "{self._dsn}".')
return self._pool
@retry(3, 3, True)
async def execute(self, query: str, *args):
async with self._pool.acquire(timeout=self._conn_timeout) as conn:
logging.info(f'Executing query on "{conn._addr[0]}:{conn._addr[1]}".')
return await conn.execute(query, *args, timeout=self._command_timeout)
@retry(3, 3, True)
async def fetch(self, query: str, *args):
async with self._pool.acquire(timeout=self._conn_timeout) as conn:
logging.info(f'Fetching from "{conn._addr[0]}:{conn._addr[1]}".')
return await conn.fetch(query, *args, timeout=self._command_timeout)
async def close(self):
if self._pool:
await self._pool.close()
self._pool = None
logging.info('Connection pool closed.')Here are the logs for execute():
[2026-03-03 17:13:39,259] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.execute().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 88, in execute
return await conn.execute(query, *args, timeout=self._command_timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 352, in execute
_, status, _ = await self._execute(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 1864, in _execute
result, _ = await self.__execute(
^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 1961, in __execute
result, stmt = await self._do_execute(
^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2027, in _do_execute
result = await executor(stmt, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "asyncpg/protocol/protocol.pyx", line 206, in bind_execute
asyncpg.exceptions.ConnectionDoesNotExistError: connection was closed in the middle of operation
[2026-03-03 17:13:39,260] WARNING [root] [pg_client.py/wrapper:64] Retrying AsyncPGClient.execute() in 3s. Attempt: 1/3.
[2026-03-03 17:13:52,262] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.execute().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/_asyncio_compat.py", line 72, in wait_for
return await fut
^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 849, in _acquire_impl
proxy = await ch.acquire() # type: PoolConnectionProxy
^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 140, in acquire
await self.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 86, in execute
async with self._pool.acquire(timeout=self._conn_timeout) as conn:
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 1024, in __aenter__
self.connection = await self.pool._acquire(self.timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 866, in _acquire
return await compat.wait_for(
^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/_asyncio_compat.py", line 71, in wait_for
async with timeout_ctx(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError
[2026-03-03 17:13:52,271] WARNING [root] [pg_client.py/wrapper:64] Retrying AsyncPGClient.execute() in 3s. Attempt: 2/3.
[2026-03-03 17:14:05,273] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.execute().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/_asyncio_compat.py", line 72, in wait_for
return await fut
^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 849, in _acquire_impl
proxy = await ch.acquire() # type: PoolConnectionProxy
^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 140, in acquire
await self.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 86, in execute
async with self._pool.acquire(timeout=self._conn_timeout) as conn:
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 1024, in __aenter__
self.connection = await self.pool._acquire(self.timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 866, in _acquire
return await compat.wait_for(
^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/_asyncio_compat.py", line 71, in wait_for
async with timeout_ctx(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError
[2026-03-03 17:14:05,274] WARNING [root] [pg_client.py/wrapper:64] Retrying AsyncPGClient.execute() in 3s. Attempt: 3/3.
[2026-03-03 17:14:18,276] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.execute().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/_asyncio_compat.py", line 72, in wait_for
return await fut
^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 849, in _acquire_impl
proxy = await ch.acquire() # type: PoolConnectionProxy
^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 140, in acquire
await self.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 86, in execute
async with self._pool.acquire(timeout=self._conn_timeout) as conn:
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 1024, in __aenter__
self.connection = await self.pool._acquire(self.timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 866, in _acquire
return await compat.wait_for(
^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/_asyncio_compat.py", line 71, in wait_for
async with timeout_ctx(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError
And logs for recreating pool:
[2026-03-03 17:14:18,278] INFO [root] [pg_client.py/close:100] Connection pool closed.
[2026-03-03 17:14:18,278] INFO [root] [pg_client.py/connect:74] Creating connection pool for "postgresql://some_host_1:5433,some_host_2:5433/some_db_name".
[2026-03-03 17:14:28,280] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.connect().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 75, in connect
self._pool = await asyncpg.create_pool(dsn=CFG.database.dsn,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 418, in _async__init__
await self._initialize()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 445, in _initialize
await first_ch.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2420, in connect
async with compat.timeout(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError
[2026-03-03 17:14:28,281] WARNING [root] [pg_client.py/wrapper:64] Retrying AsyncPGClient.connect() in 3s. Attempt: 1/3.
[2026-03-03 17:14:31,281] INFO [root] [pg_client.py/connect:74] Creating connection pool for "postgresql://some_host_1:5433,some_host_2:5433/some_db_name".
[2026-03-03 17:14:41,282] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.connect().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 75, in connect
self._pool = await asyncpg.create_pool(dsn=CFG.database.dsn,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 418, in _async__init__
await self._initialize()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 445, in _initialize
await first_ch.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2420, in connect
async with compat.timeout(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError
[2026-03-03 17:14:41,283] WARNING [root] [pg_client.py/wrapper:64] Retrying AsyncPGClient.connect() in 3s. Attempt: 2/3.
[2026-03-03 17:14:44,283] INFO [root] [pg_client.py/connect:74] Creating connection pool for "postgresql://some_host_1:5433,some_host_2:5433/some_db_name".
[2026-03-03 17:14:54,284] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.connect().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 75, in connect
self._pool = await asyncpg.create_pool(dsn=CFG.database.dsn,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 418, in _async__init__
await self._initialize()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 445, in _initialize
await first_ch.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2420, in connect
async with compat.timeout(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError
[2026-03-03 17:14:54,285] WARNING [root] [pg_client.py/wrapper:64] Retrying AsyncPGClient.connect() in 3s. Attempt: 3/3.
[2026-03-03 17:14:57,286] INFO [root] [pg_client.py/connect:74] Creating connection pool for "postgresql://some_host_1:5433,some_host_2:5433/some_db_name".
[2026-03-03 17:15:07,287] ERROR [root] [pg_client.py/wrapper:56] Failed to AsyncPGClient.connect().
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2421, in connect
return await connect_utils._connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 1049, in _connect
conn = await _connect_addr(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 886, in _connect_addr
return await __connect_addr(params, True, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 931, in __connect_addr
tr, pr = await connector
^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connect_utils.py", line 802, in _create_ssl_connection
tr, pr = await loop.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 53, in wrapper
return await func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/db_adapters/pg_client.py", line 75, in connect
self._pool = await asyncpg.create_pool(dsn=CFG.database.dsn,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 418, in _async__init__
await self._initialize()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 445, in _initialize
await first_ch.connect()
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 132, in connect
self._con = await self._pool._get_new_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/pool.py", line 517, in _get_new_connection
con = await self._connect(
^^^^^^^^^^^^^^^^^^^^
File "/Users/22610520/_Mika/Projects/Python/chart_maker_service/.venv/lib/python3.11/site-packages/asyncpg/connection.py", line 2420, in connect
async with compat.timeout(timeout):
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/timeouts.py", line 115, in __aexit__
raise TimeoutError from exc_val
TimeoutError