Dependencies: Update requirement to psycopg~=3.0 (#6362)

The `psycopg` library is used by `sqlalchemy` to connect to the
PostgreSQL server. So far, the `psycopg2` package was used, but the
next generation v3 has already been out for a while. The release comes
with a number of performance improvements according to the author.
Although there is no clear timeline for support of v2, we are not
waiting and switching now to the new version.

When v2 was released, instead of releasing a new major version, it was
released as a new library changing the name from `psycopg` to `psycopg2`
but for v3 they are switching back to `psycopg`. This means that users
would still be able to run v2 and v3 along side one another in a single
Python environment. This supports the decision to move to v3.

Note that `aiida-core` will not be supporting both at the same time and
from now only supports v3. Interestingly, from the very early versions
of AiiDA, the profile would contain the `database_engine` key. This is
still an option in `verdi setup` and `verdi quicksetup`, except it is
not really an option as it is hardcoded to `postgresql_psycopg2`.
So all `core.psql_dos` profiles out there contain:

    'main': {
        'storage': {
            'backend': 'core.psql_dos',
            'config': {
                'database_engine': 'postgresql_psycopg2',
                ...
            }
        },
    }

The value is not actually used however as the connection string for
sqlalchemy's engine, where it _could_ be used, simply hardcodes this to
`postgres://` which is the default psycopg dialect and maps to
`postgres+psycopg2://`. This value is now simply updated to
`postgres+psycopg://` to target the new version of `psycopg`.

Because it is hardcoded, a migration for the existing configs is not
required and it can be left as a vestigial attribute. Since `verdi setup`
and `verdi quicksetup` are deprecated now anyway, the options don't have
to be removed here.

.github/config/profile.yaml

@@ -4,7 +4,7 @@ first_name: Giuseppe
 last_name: Verdi
 institution: Khedivial
 db_backend: core.psql_dos
-db_engine: postgresql_psycopg2
+db_engine: postgresql_psycopg
 db_host: localhost
 db_port: 5432
 db_name: test_aiida

docs/source/howto/installation.rst

@@ -55,7 +55,7 @@ To display these parameters, use ``verdi profile show``:
     storage:
         backend: core.psql_dos
         config:
-            database_engine: postgresql_psycopg2
+            database_engine: postgresql_psycopg
             database_hostname: localhost
             database_name: name
             database_password: abc

docs/source/installation/docker.rst

@@ -66,7 +66,7 @@ which should show something like::
     ✔ version:     AiiDA v2.5.1
     ✔ config:      /home/aiida/.aiida
     ✔ profile:     default
-    ✔ storage:     Storage for 'default' [open] @ postgresql://aiida:***@localhost:5432
+    ✔ storage:     Storage for 'default' [open] @ postgresql+psycopg://aiida:***@localhost:5432
     ✔ rabbitmq:    Connected to RabbitMQ v3.10.18 as amqp://guest:[email protected]:5672
     ✔ daemon:      Daemon is running with PID 324
 

docs/source/installation/troubleshooting.rst

@@ -13,7 +13,7 @@ If you experience any problems, first check that all services are up and running
    ✓ version:     AiiDA v2.0.0
    ✓ config:      /path/to/.aiida
    ✓ profile:     default
-   ✓ storage:     Storage for 'default' @ postgresql://username:***@localhost:5432/db_name / file:///path/to/repository
+   ✓ storage:     Storage for 'default' @ postgresql+psycopg://username:***@localhost:5432/db_name / file:///path/to/repository
    ✓ rabbitmq:    Connected as amqp://127.0.0.1?heartbeat=600
    ✓ daemon:      Daemon is running as PID 2809 since 2019-03-15 16:27:52
 

docs/source/nitpick-exceptions

@@ -236,8 +236,6 @@ py:class yaml.nodes.MappingNode
 py:class yaml.nodes.ScalarNode
 py:class uuid.UUID
 
-py:class psycopg2.extensions.cursor
-
 py:class alembic.config.Config
 py:class alembic.op
 py:class alembic.runtime.migration.MigrationContext

docs/source/reference/command_line.rst

@@ -427,7 +427,7 @@ Below is a list with all available subcommands.
       --first-name NONEMPTYSTRING     First name of the user.  [required]
       --last-name NONEMPTYSTRING      Last name of the user.  [required]
       --institution NONEMPTYSTRING    Institution of the user.  [required]
-      --db-engine [postgresql_psycopg2]
+      --db-engine [postgresql_psycopg]
                                       Engine to use to connect to the database.  [required]
       --db-backend [core.psql_dos]    Database backend to use.  [required]
       --db-host HOSTNAME              Database server host. Leave empty for "peer"
@@ -534,7 +534,7 @@ Below is a list with all available subcommands.
       --first-name NONEMPTYSTRING     First name of the user.  [required]
       --last-name NONEMPTYSTRING      Last name of the user.  [required]
       --institution NONEMPTYSTRING    Institution of the user.  [required]
-      --db-engine [postgresql_psycopg2]
+      --db-engine [postgresql_psycopg]
                                       Engine to use to connect to the database.  [required]
       --db-backend [core.psql_dos]    Database backend to use.  [required]
       --db-host HOSTNAME              Database server host. Leave empty for "peer"

environment.yml

@@ -23,9 +23,9 @@ dependencies:
 - numpy~=1.21
 - paramiko>=2.7.2,~=2.7
 - plumpy~=0.22.3
-- pgsu~=0.2.1
+- pgsu~=0.3.0
 - psutil~=5.6
-- psycopg2-binary~=2.8
+- psycopg[binary]~=3.0
 - pydantic~=2.4
 - pytz~=2021.1
 - pyyaml~=6.0

pyproject.toml

@@ -35,9 +35,9 @@ dependencies = [
   'numpy~=1.21',
   'paramiko~=2.7,>=2.7.2',
   'plumpy~=0.22.3',
-  'pgsu~=0.2.1',
+  'pgsu~=0.3.0',
   'psutil~=5.6',
-  'psycopg2-binary~=2.8',
+  'psycopg[binary]~=3.0',
   'pydantic~=2.4',
   'pytz~=2021.1',
   'pyyaml~=6.0',
@@ -326,7 +326,7 @@ module = [
   'pgtest.*',
   'phonopy.*',
   'psutil.*',
-  'psycopg2.*',
+  'psycopg.*',
   'pymatgen.*',
   'pymysql.*',
   'pyparsing.*',

requirements/requirements-py-3.10.txt

@@ -113,7 +113,7 @@ paramiko==2.12.0
 parso==0.8.3
 pexpect==4.8.0
 pg8000==1.29.8
-pgsu==0.2.3
+pgsu==0.3.0
 pgtest==1.3.2
 pickleshare==0.7.5
 pillow==9.5.0
@@ -124,7 +124,7 @@ plumpy==0.22.3
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
 psutil==5.9.5
-psycopg2-binary==2.9.6
+psycopg[binary]==3.1.18
 ptyprocess==0.7.0
 pure-eval==0.2.2
 py-cpuinfo==9.0.0

requirements/requirements-py-3.11.txt

@@ -112,7 +112,7 @@ paramiko==2.12.0
 parso==0.8.3
 pexpect==4.8.0
 pg8000==1.29.8
-pgsu==0.2.3
+pgsu==0.3.0
 pgtest==1.3.2
 pickleshare==0.7.5
 pillow==9.5.0
@@ -123,7 +123,7 @@ plumpy==0.22.3
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
 psutil==5.9.5
-psycopg2-binary==2.9.6
+psycopg[binary]==3.1.18
 ptyprocess==0.7.0
 pure-eval==0.2.2
 py-cpuinfo==9.0.0

requirements/requirements-py-3.12.txt

@@ -112,7 +112,7 @@ paramiko==2.12.0
 parso==0.8.3
 pexpect==4.8.0
 pg8000==1.30.2
-pgsu==0.2.4
+pgsu==0.3.0
 pgtest==1.3.2
 pickleshare==0.7.5
 pillow==10.1.0
@@ -123,7 +123,7 @@ plumpy==0.22.3
 prometheus-client==0.17.1
 prompt-toolkit==3.0.39
 psutil==5.9.6
-psycopg2-binary==2.9.9
+psycopg[binary]==3.1.18
 ptyprocess==0.7.0
 pure-eval==0.2.2
 py-cpuinfo==9.0.0

requirements/requirements-py-3.9.txt

@@ -115,7 +115,7 @@ paramiko==2.12.0
 parso==0.8.3
 pexpect==4.8.0
 pg8000==1.29.8
-pgsu==0.2.3
+pgsu==0.3.0
 pgtest==1.3.2
 pickleshare==0.7.5
 pillow==9.5.0
@@ -126,7 +126,7 @@ plumpy==0.22.3
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
 psutil==5.9.5
-psycopg2-binary==2.9.6
+psycopg[binary]==3.1.18
 ptyprocess==0.7.0
 pure-eval==0.2.2
 py-cpuinfo==9.0.0

src/aiida/cmdline/commands/cmd_setup.py

@@ -241,8 +241,8 @@ def quicksetup(
         'db_backend': db_backend,
         'db_name': db_name,
         # from now on we connect as the AiiDA DB user, which may be forbidden when going via sockets
-        'db_host': postgres.host_for_psycopg2,
-        'db_port': postgres.port_for_psycopg2,
+        'db_host': postgres.host_for_psycopg,
+        'db_port': postgres.port_for_psycopg,
         'db_username': db_username,
         'db_password': db_password,
         'broker_protocol': broker_protocol,

src/aiida/cmdline/params/options/commands/setup.py

@@ -258,7 +258,7 @@ def get_quicksetup_password(ctx, param, value):
     '--su-db-name',
     help='Name of the template database to connect to as the database superuser.',
     type=click.STRING,
-    default=DEFAULT_DBINFO['database'],
+    default=DEFAULT_DBINFO['dbname'],
 )
 
 QUICKSETUP_SUPERUSER_DATABASE_PASSWORD = options.OverridableOption(
@@ -288,7 +288,7 @@ def get_quicksetup_password(ctx, param, value):
 SETUP_DATABASE_ENGINE = QUICKSETUP_DATABASE_ENGINE.clone(
     prompt='Database engine',
     contextual_default=functools.partial(
-        get_profile_attribute_default, ('storage.config.database_engine', 'postgresql_psycopg2')
+        get_profile_attribute_default, ('storage.config.database_engine', 'postgresql_psycopg')
     ),
     cls=options.interactive.InteractiveOption,
 )

src/aiida/cmdline/params/options/main.py

@@ -383,8 +383,8 @@ def set_log_level(ctx, _param, value):
     '--db-engine',
     required=True,
     help='Engine to use to connect to the database.',
-    default='postgresql_psycopg2',
-    type=click.Choice(['postgresql_psycopg2']),
+    default='postgresql_psycopg',
+    type=click.Choice(['postgresql_psycopg']),
 )
 
 DB_BACKEND = OverridableOption(

src/aiida/manage/external/postgres.py

@@ -96,7 +96,7 @@ def create_dbuser(self, dbuser, dbpass, privileges=''):
 
         :param str dbuser: Name of the user to be created.
         :param str dbpass: Password the user should be given.
-        :raises: psycopg2.errors.DuplicateObject if user already exists and
+        :raises: psycopg.errors.DuplicateObject if user already exists and
             self.connection_mode == PostgresConnectionMode.PSYCOPG
         """
         self.execute(_CREATE_USER_COMMAND.format(dbuser, dbpass, privileges))
@@ -130,14 +130,13 @@ def find_new_dbuser(self, start_from='aiida'):
     def can_user_authenticate(self, dbuser, dbpass):
         """Check whether the database user credentials are valid.
 
-        Checks whether dbuser has access to the `template1` postgres database
-        via psycopg2.
+        Checks whether dbuser has access to the `template1` postgres database via psycopg.
 
         :param dbuser: the database user
         :param dbpass: the database password
         :return: True if the credentials are valid, False otherwise
         """
-        import psycopg2
+        import psycopg
         from pgsu import _execute_psyco
 
         dsn = self.dsn.copy()
@@ -146,7 +145,7 @@ def can_user_authenticate(self, dbuser, dbpass):
 
         try:
             _execute_psyco('SELECT 1', dsn)
-        except psycopg2.OperationalError:
+        except psycopg.OperationalError:
             return False
 
         return True
@@ -227,8 +226,8 @@ def create_dbuser_db_safe(self, dbname, dbuser, dbpass):
         return dbuser, dbname
 
     @property
-    def host_for_psycopg2(self):
-        """Return correct host for psycopg2 connection (as required by regular AiiDA operation)."""
+    def host_for_psycopg(self):
+        """Return correct host for psycopg connection (as required by regular AiiDA operation)."""
         host = self.dsn.get('host')
         if self.connection_mode == PostgresConnectionMode.PSQL:
             # If "sudo su postgres" was needed to create the DB, we are likely on Ubuntu, where
@@ -238,8 +237,8 @@ def host_for_psycopg2(self):
         return host
 
     @property
-    def port_for_psycopg2(self):
-        """Return port for psycopg2 connection (as required by regular AiiDA operation)."""
+    def port_for_psycopg(self):
+        """Return port for psycopg connection (as required by regular AiiDA operation)."""
         return self.dsn.get('port')
 
     @property

src/aiida/manage/tests/pytest_fixtures.py

@@ -96,7 +96,7 @@ def create_database(
         from aiida.manage.external.postgres import Postgres
 
         postgres_config = {
-            'database_engine': 'postgresql_psycopg2',
+            'database_engine': 'postgresql_psycopg',
             'database_name': database_name or str(uuid.uuid4()),
             'database_username': database_username or 'guest',
             'database_password': database_password or 'guest',
@@ -109,8 +109,8 @@ def create_database(
             )
         postgres.create_db(postgres_config['database_username'], postgres_config['database_name'])
 
-        postgres_config['database_hostname'] = postgres.host_for_psycopg2
-        postgres_config['database_port'] = postgres.port_for_psycopg2
+        postgres_config['database_hostname'] = postgres.host_for_psycopg
+        postgres_config['database_port'] = postgres.port_for_psycopg
 
         return postgres_config
 

src/aiida/restapi/common/utils.py

@@ -9,7 +9,7 @@
 """Util methods"""
 
 import urllib.parse
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from flask import jsonify
 from flask.json.provider import DefaultJSONProvider
@@ -670,7 +670,6 @@ def parse_query_string(self, query_string):
         :param query_string (as obtained from request.query_string)
         :return: parsed values for the querykeys
         """
-        from psycopg2.tz import FixedOffsetTimezone
         from pyparsing import (
             Combine,
             Group,
@@ -774,10 +773,10 @@ def validate_time(toks):
             if dtobj.tzinfo is not None and dtobj.utcoffset() is not None:
                 tzoffset_minutes = int(dtobj.utcoffset().total_seconds() // 60)
                 return DatetimePrecision(
-                    dtobj.replace(tzinfo=FixedOffsetTimezone(offset=tzoffset_minutes, name=None)), precision
+                    dtobj.replace(tzinfo=timezone(offset=timedelta(minutes=tzoffset_minutes), name='UTC')), precision
                 )
 
-            return DatetimePrecision(dtobj.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)), precision)
+            return DatetimePrecision(dtobj.replace(tzinfo=timezone(offset=timedelta(minutes=0), name='UTC')), precision)
 
         ########################################################################
 

src/aiida/storage/psql_dos/backend.py

@@ -80,7 +80,7 @@ class Model(BaseModel, defer_build=True):
         database_engine: str = Field(
             title='PostgreSQL engine',
             description='The engine to use to connect to the database.',
-            default='postgresql_psycopg2',
+            default='postgresql_psycopg',
         )
         database_hostname: str = Field(
             title='PostgreSQL hostname', description='The hostname of the PostgreSQL server.', default='localhost'

src/aiida/storage/psql_dos/utils.py

@@ -42,7 +42,7 @@ def create_sqlalchemy_engine(config: PsqlConfig):
     separator = ':' if config['database_port'] else ''
     password = quote_plus(config['database_password'])
 
-    engine_url = 'postgresql://{user}:{password}@{hostname}{separator}{port}/{name}'.format(
+    engine_url = 'postgresql+psycopg://{user}:{password}@{hostname}{separator}{port}/{name}'.format(
         separator=separator,
         user=config['database_username'],
         password=password,

src/aiida/tools/pytest_fixtures/storage.py

@@ -41,7 +41,7 @@ def create_database(
             self._create()
 
         postgres_config = {
-            'database_engine': 'postgresql_psycopg2',
+            'database_engine': 'postgresql_psycopg',
             'database_name': database_name or str(uuid4()),
             'database_username': database_username or 'guest',
             'database_password': database_password or 'guest',
@@ -54,8 +54,8 @@ def create_database(
             )
         postgres.create_db(postgres_config['database_username'], postgres_config['database_name'])
 
-        postgres_config['database_hostname'] = postgres.host_for_psycopg2
-        postgres_config['database_port'] = postgres.port_for_psycopg2
+        postgres_config['database_hostname'] = postgres.host_for_psycopg
+        postgres_config['database_port'] = postgres.port_for_psycopg
 
         return postgres_config
 

tests/conftest.py

@@ -351,7 +351,7 @@ def _create_profile(name='test-profile', **kwargs):
             'storage': {
                 'backend': kwargs.pop('storage_backend', 'core.psql_dos'),
                 'config': {
-                    'database_engine': kwargs.pop('database_engine', 'postgresql_psycopg2'),
+                    'database_engine': kwargs.pop('database_engine', 'postgresql_psycopg'),
                     'database_hostname': kwargs.pop('database_hostname', 'localhost'),
                     'database_port': kwargs.pop('database_port', 5432),
                     'database_name': kwargs.pop('database_name', name),

tests/orm/test_querybuilder/test_as_sql.txt

@@ -1 +1 @@
-'SELECT db_dbnode_1.uuid \nFROM db_dbnode AS db_dbnode_1 \nWHERE CAST(db_dbnode_1.node_type AS VARCHAR) LIKE %(param_1)s AND CASE WHEN (jsonb_typeof((db_dbnode_1.extras #> %(extras_1)s)) = %(jsonb_typeof_1)s) THEN (db_dbnode_1.extras #>> %(extras_1)s) = %(param_2)s ELSE %(param_3)s END' % {'param_1': '%', 'extras_1': ('tag4',), 'jsonb_typeof_1': 'string', 'param_2': 'appl_pecoal', 'param_3': False}
+'SELECT db_dbnode_1.uuid \nFROM db_dbnode AS db_dbnode_1 \nWHERE CAST(db_dbnode_1.node_type AS VARCHAR) LIKE %(param_1)s::VARCHAR AND CASE WHEN (jsonb_typeof((db_dbnode_1.extras #> %(extras_1)s)) = %(jsonb_typeof_1)s::VARCHAR) THEN (db_dbnode_1.extras #>> %(extras_1)s) = %(param_2)s::VARCHAR ELSE %(param_3)s END' % {'param_1': '%', 'extras_1': ('tag4',), 'jsonb_typeof_1': 'string', 'param_2': 'appl_pecoal', 'param_3': False}