Release 4.1.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.1
+current_version = 4.1.0-rc8
 tag_name = {new_version}
 commit = True
 tag = True

aleph/core.py

@@ -23,16 +23,12 @@
 from werkzeug.local import LocalProxy
 from werkzeug.middleware.profiler import ProfilerMiddleware
 
-from aleph import __version__ as aleph_version
 from aleph.settings import SETTINGS
 from aleph.cache import Cache
 from aleph.oauth import configure_oauth
 from aleph.util import LoggingTransport
 from aleph.metrics.flask import PrometheusExtension
 
-import sentry_sdk
-from sentry_sdk.integrations.flask import FlaskIntegration
-
 
 NONE = "'none'"
 log = logging.getLogger(__name__)
@@ -67,18 +63,6 @@ def create_app(config=None):
         config = {}
 
     configure_logging(level=logging.DEBUG)
-
-    if SETTINGS.SENTRY_DSN:
-        sentry_sdk.init(
-            dsn=SETTINGS.SENTRY_DSN,
-            integrations=[
-                FlaskIntegration(),
-            ],
-            traces_sample_rate=0,
-            release=aleph_version,
-            environment=SETTINGS.SENTRY_ENVIRONMENT,
-            send_default_pii=False,
-        )
     app = Flask("aleph")
     app.config.from_object(SETTINGS)
     app.config.update(config)

aleph/logic/api_keys.py

@@ -9,7 +9,7 @@
 from aleph.model.common import make_token
 from aleph.logic.mail import email_role
 from aleph.logic.roles import update_role
-from aleph.logic.util import ui_url
+from aleph.logic.util import ui_url, hash_api_key
 
 # Number of days after which API keys expire
 API_KEY_EXPIRATION_DAYS = 90
@@ -29,15 +29,16 @@ def generate_user_api_key(role):
     email_role(role, subject, html=html, plain=plain)
 
     now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
-    role.api_key = make_token()
+    api_key = make_token()
+    role.api_key_digest = hash_api_key(api_key)
     role.api_key_expires_at = now + datetime.timedelta(days=API_KEY_EXPIRATION_DAYS)
     role.api_key_expiration_notification_sent = None
 
     db.session.add(role)
     db.session.commit()
     update_role(role)
 
-    return role.api_key
+    return api_key
 
 
 def send_api_key_expiration_notifications():
@@ -70,7 +71,7 @@ def _send_api_key_expiration_notification(
     query = query.where(
         and_(
             and_(
-                Role.api_key != None,  # noqa: E711
+                Role.api_key_digest != None,  # noqa: E711
                 func.date(Role.api_key_expires_at) <= threshold,
             ),
             or_(
@@ -104,10 +105,31 @@ def reset_api_key_expiration():
     query = query.yield_per(500)
     query = query.where(
         and_(
-            Role.api_key != None,  # noqa: E711
+            Role.api_key_digest != None,  # noqa: E711
             Role.api_key_expires_at == None,  # noqa: E711
         )
     )
 
     query.update({Role.api_key_expires_at: expires_at})
     db.session.commit()
+
+
+def hash_plaintext_api_keys():
+    query = Role.all_users()
+    query = query.yield_per(250)
+    query = query.where(
+        and_(
+            Role.api_key != None,  # noqa: E711
+            Role.api_key_digest == None,  # noqa: E711
+        )
+    )
+
+    results = db.session.execute(query).scalars()
+
+    for index, partition in enumerate(results.partitions()):
+        for role in partition:
+            role.api_key_digest = hash_api_key(role.api_key)
+            db.session.add(role)
+            log.info(f"Hashing API key: {role}")
+        log.info(f"Comitting partition {index}")
+        db.session.commit()

aleph/logic/util.py

@@ -1,4 +1,5 @@
 import jwt
+import hashlib
 from normality import ascii_text
 from urllib.parse import urlencode, urljoin
 from datetime import datetime, timedelta
@@ -58,3 +59,11 @@ def archive_token(token):
     token = jwt.decode(token, key=SETTINGS.SECRET_KEY, algorithms=DECODE, verify=True)
     expire = datetime.utcfromtimestamp(token["exp"])
     return token.get("c"), token.get("f"), token.get("m"), expire
+
+
+def hash_api_key(api_key):
+    if api_key is None:
+        return None
+
+    digest = hashlib.sha256(api_key.encode("utf-8")).hexdigest()
+    return f"sha256${digest}"

aleph/manage.py

@@ -20,7 +20,10 @@
 from aleph.queues import get_status, cancel_queue
 from aleph.queues import get_active_dataset_status
 from aleph.index.admin import delete_index
-from aleph.logic.api_keys import reset_api_key_expiration as _reset_api_key_expiration
+from aleph.logic.api_keys import (
+    reset_api_key_expiration as _reset_api_key_expiration,
+    hash_plaintext_api_keys as _hash_plaintext_api_keys,
+)
 from aleph.index.entities import iter_proxies
 from aleph.index.util import AlephOperationalException
 from aleph.logic.collections import create_collection, update_collection
@@ -573,3 +576,9 @@ def evilshit():
 def reset_api_key_expiration():
     """Reset the expiration date of all legacy, non-expiring API keys."""
     _reset_api_key_expiration()
+
+
+@cli.command()
+def hash_plaintext_api_keys():
+    """Hash legacy plaintext API keys."""
+    _hash_plaintext_api_keys()

aleph/migrate/versions/31e24765dee3_add_api_key_digest_column.py

@@ -0,0 +1,28 @@
+"""Add api_key_digest column
+
+Revision ID: 31e24765dee3
+Revises: d46fc882ec6b
+Create Date: 2024-07-04 11:07:19.915782
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "31e24765dee3"
+down_revision = "d46fc882ec6b"
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column("role", sa.Column("api_key_digest", sa.Unicode()))
+    op.create_index(
+        index_name="ix_role_api_key_digest",
+        table_name="role",
+        columns=["api_key_digest"],
+        unique=True,
+    )
+
+
+def downgrade():
+    op.drop_column("role", "api_key_digest")

aleph/model/role.py

@@ -1,14 +1,15 @@
 import logging
 from datetime import datetime, timezone
 from normality import stringify
-from sqlalchemy import or_, not_, func
+from sqlalchemy import and_, or_, not_, func
 from itsdangerous import URLSafeTimedSerializer
 from werkzeug.security import generate_password_hash, check_password_hash
 
 from aleph.core import db
 from aleph.settings import SETTINGS
 from aleph.model.common import SoftDeleteModel, IdModel, query_like
 from aleph.util import anonymize_email
+from aleph.logic.util import hash_api_key
 
 log = logging.getLogger(__name__)
 
@@ -52,6 +53,7 @@ class Role(db.Model, IdModel, SoftDeleteModel):
     email = db.Column(db.Unicode, nullable=True)
     type = db.Column(db.Enum(*TYPES, name="role_type"), nullable=False)
     api_key = db.Column(db.Unicode, nullable=True)
+    api_key_digest = db.Column(db.Unicode, nullable=True)
     api_key_expires_at = db.Column(db.DateTime, nullable=True)
     api_key_expiration_notification_sent = db.Column(db.Integer, nullable=True)
     is_admin = db.Column(db.Boolean, nullable=False, default=False)
@@ -72,7 +74,7 @@ def has_password(self):
 
     @property
     def has_api_key(self):
-        return self.api_key is not None
+        return self.api_key_digest is not None
 
     @property
     def is_public(self):
@@ -195,12 +197,20 @@ def by_email(cls, email):
 
     @classmethod
     def by_api_key(cls, api_key):
-        if api_key is None:
+        if api_key is None or not len(api_key.strip()):
             return None
+
         q = cls.all()
-        q = q.filter_by(api_key=api_key)
-        utcnow = datetime.now(timezone.utc)
 
+        digest = hash_api_key(api_key)
+        q = q.filter(
+            and_(
+                cls.api_key_digest != None,  # noqa: E711
+                cls.api_key_digest == digest,
+            )
+        )
+
+        utcnow = datetime.now(timezone.utc)
         # TODO: Exclude API keys without expiration date after deadline
         # See https://github.com/alephdata/aleph/issues/3729
         q = q.filter(
@@ -212,6 +222,7 @@ def by_api_key(cls, api_key):
 
         q = q.filter(cls.type == cls.USER)
         q = q.filter(cls.is_blocked == False)  # noqa
+
         return q.first()
 
     @classmethod

aleph/tests/test_api_keys.py

@@ -5,33 +5,35 @@
 from aleph.logic.api_keys import (
     generate_user_api_key,
     send_api_key_expiration_notifications,
+    hash_plaintext_api_keys,
 )
+from aleph.logic.util import hash_api_key
 from aleph.tests.util import TestCase
 
 
 class ApiKeysTestCase(TestCase):
     def test_generate_user_api_key(self):
         role = self.create_user()
-        assert role.api_key is None
+        assert role.api_key_digest is None
         assert role.api_key_expires_at is None
 
         with time_machine.travel("2024-01-01T00:00:00Z"):
             generate_user_api_key(role)
             db.session.refresh(role)
-            assert role.api_key is not None
+            assert role.api_key_digest is not None
             assert role.api_key_expires_at.date() == datetime.date(2024, 3, 31)
 
-        old_key = role.api_key
+        old_digest = role.api_key_digest
 
         with time_machine.travel("2024-02-01T00:00:00Z"):
             generate_user_api_key(role)
             db.session.refresh(role)
-            assert role.api_key != old_key
+            assert role.api_key_digest != old_digest
             assert role.api_key_expires_at.date() == datetime.date(2024, 5, 1)
 
     def test_generate_user_api_key_notification(self):
         role = self.create_user(email="[email protected]")
-        assert role.api_key is None
+        assert role.api_key_digest is None
 
         with mail.record_messages() as outbox:
             assert len(outbox) == 0
@@ -65,7 +67,7 @@ def test_send_api_key_expiration_notifications(self):
                 assert len(outbox) == 1
                 assert outbox[0].subject == "[Aleph] API key generated"
 
-                assert role.api_key is not None
+                assert role.api_key_digest is not None
                 assert role.api_key_expires_at.date() == datetime.date(2024, 3, 31)
 
                 assert len(outbox) == 1
@@ -122,7 +124,7 @@ def test_send_api_key_expiration_notifications(self):
 
     def test_send_api_key_expiration_notifications_no_key(self):
         role = self.create_user(email="[email protected]")
-        assert role.api_key is None
+        assert role.api_key_digest is None
 
         with mail.record_messages() as outbox:
             assert len(outbox) == 0
@@ -193,3 +195,30 @@ def test_send_api_key_expiration_notifications_regenerate(self):
 
                 assert outbox[4].subject == "[Aleph] Your API key will expire in 7 days"
                 assert outbox[5].subject == "[Aleph] Your API key has expired"
+
+    def test_hash_plaintext_api_keys(self):
+        user_1 = self.create_user(foreign_id="user_1", email="[email protected]")
+        user_1.api_key = "1234567890"
+        user_1.api_key_digest = None
+
+        user_2 = self.create_user(foreign_id="user_2", email="[email protected]")
+        user_2.api_key = None
+        user_2.api_key_digest = None
+
+        db.session.add_all([user_1, user_2])
+        db.session.commit()
+
+        hash_plaintext_api_keys()
+
+        db.session.refresh(user_1)
+
+        # Do not delete the plaintext API key to allow for version rollbacks.
+        # `api_key` column will be removed in the next version at which point all
+        # plaintext keys will be deleted.
+        assert user_1.api_key == "1234567890"
+
+        assert user_1.api_key_digest == hash_api_key("1234567890")
+
+        db.session.refresh(user_2)
+        assert user_2.api_key is None
+        assert user_2.api_key_digest is None

aleph/tests/test_role_model.py

@@ -5,6 +5,7 @@
 from aleph.model import Role
 from aleph.tests.factories.models import RoleFactory
 from aleph.logic.roles import create_user, create_group
+from aleph.logic.util import hash_api_key
 
 from aleph.tests.util import TestCase
 
@@ -83,7 +84,7 @@ def test_remove_role(self):
 
     def test_role_by_api_key(self):
         role_ = self.create_user()
-        role_.api_key = "1234567890"
+        role_.api_key_digest = hash_api_key("1234567890")
         db.session.add(role_)
         db.session.commit()
 
@@ -93,7 +94,7 @@ def test_role_by_api_key(self):
 
     def test_role_by_api_key_empty(self):
         role_ = self.create_user()
-        assert role_.api_key is None
+        assert role_.api_key_digest is None
 
         role = Role.by_api_key(None)
         assert role is None
@@ -103,26 +104,26 @@ def test_role_by_api_key_empty(self):
 
     def test_role_by_api_key_expired(self):
         role_ = self.create_user()
-        role_.api_key = "1234567890"
+        role_.api_key_digest = hash_api_key("1234567890")
         role_.api_key_expires_at = datetime.datetime(2024, 3, 31, 0, 0, 0)
         db.session.add(role_)
         db.session.commit()
 
         with time_machine.travel("2024-03-30T23:59:59Z"):
             print(role_.api_key_expires_at)
-            role = Role.by_api_key(role_.api_key)
+            role = Role.by_api_key("1234567890")
             assert role is not None
             assert role.id == role_.id
 
         with time_machine.travel("2024-03-31T00:00:00Z"):
-            role = Role.by_api_key(role_.api_key)
+            role = Role.by_api_key("1234567890")
             assert role is None
 
     def test_role_by_api_key_legacy_without_expiration(self):
         # Ensure that legacy API keys that were created without an expiration
         # date continue to work.
         role_ = self.create_user()
-        role_.api_key = "1234567890"
+        role_.api_key_digest = hash_api_key("1234567890")
         role_.api_key_expires_at = None
         db.session.add(role_)
         db.session.commit()