Gracefully handle exhausted postgresql connection slots
The default configuration of postgresql has max_connections = 100. The default configuration of Debusine seems to exceed that value under load. Typical traceback:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 289, in ensure_connection
self.connect()
^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 270, in connect
self.connection = self.get_new_connection(conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/postgresql/base.py", line 275, in get_new_connection
connection = self.Database.connect(**conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The above exception (connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: FATAL: remaining connection slots are reserved for non-replication superuser
+connections
) was the direct cause of the following exception:
File "/usr/lib/python3/dist-packages/django/core/handlers/exception.py", line 55, in inner
response = get_response(request)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/debusine/server/middlewares/scopes.py", line 66, in __call__
context.set_scope(self.get_scope(scope_name))
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/debusine/server/middlewares/scopes.py", line 77, in get_scope
return get_object_or_404(Scope, name=name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/shortcuts.py", line 85, in get_object_or_404
return queryset.get(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 633, in get
num = len(clone)
^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 380, in __len__
self._fetch_all()
^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 1881, in _fetch_all
self._result_cache = list(self._iterable_class(self))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 91, in __iter__
results = compiler.execute_sql(
File "/usr/lib/python3/dist-packages/django/db/models/sql/compiler.py", line 1560, in execute_sql
cursor = self.connection.cursor()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 330, in cursor
return self._cursor()
^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 306, in _cursor
self.ensure_connection()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 288, in ensure_connection
with self.wrap_database_errors:
^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/utils.py", line 91, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 289, in ensure_connection
self.connect()
^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/base/base.py", line 270, in connect
self.connection = self.get_new_connection(conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/django/db/backends/postgresql/base.py", line 275, in get_new_connection
connection = self.Database.connect(**conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Exception Type: OperationalError at ...
Exception Value: connection to server on socket "/var/run/postgresql/..." failed: FATAL: remaining connection slots are reserved for non-replication superuser connections
To make matters worse, every such invocation generates a mail.