From 6de35f746056ca6b29765308c8fc58d678170ff7 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Tue, 11 Feb 2025 18:04:35 +0100
Subject: [PATCH 1/5] Remove experimental MXID value from user notification
preferences
* since we're not quite ready for matrix notifications yet, this wasn't
supposed to be added to the code
---
invenio_config_tuw/users/templates/notifications_settings.html | 1 -
1 file changed, 1 deletion(-)
diff --git a/invenio_config_tuw/users/templates/notifications_settings.html b/invenio_config_tuw/users/templates/notifications_settings.html
index 51793b3..6d2f85c 100644
--- a/invenio_config_tuw/users/templates/notifications_settings.html
+++ b/invenio_config_tuw/users/templates/notifications_settings.html
@@ -37,7 +37,6 @@ details.
{#- change: create vars for other fields -#}
{%- set enabled_field = form.enabled %}
{%- set secondary_email_field = form.secondary_email %}
- {%- set matrix_user_id_field = form.matrix_user_id %}
{%- set notif_enabled = current_user.preferences.get("notifications", {}).get("enabled", True) %}
<div class="field">
--
GitLab
From 798907f187cdc2f80010960ac37f8ddd0a95bda2 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Tue, 11 Feb 2025 18:13:59 +0100
Subject: [PATCH 2/5] Perform config magic before extension loading
* previously, our configuration magic was performed as part of the
`finalize_app` entrypoint
* while this worked well enough for most use cases, some extensions
would break this model by caching values during their initialization
* especially caching-related extensions like to take eagerly create
clients with the current configuration items, which are hard to fix
afterwards
* keeping track of all the instantiated clients just to change their
connection details when we perform our magic really didn't seem like
a good approach
* so instead, we now made sure that our extension is loaded first and
can thus perform its configuration magic before any other extension
has a chance of caching values
* also, extension loading happens after the configuration loading is
done (especially the handling of environment variables)
---
invenio_config_tuw/config.py | 39 ++++++++++++++++++++
invenio_config_tuw/ext.py | 20 ++++++++---
invenio_config_tuw/startup/__init__.py | 3 +-
invenio_config_tuw/startup/config.py | 50 ++++++++++----------------
pyproject.toml | 17 +++++----
5 files changed, 84 insertions(+), 45 deletions(-)
diff --git a/invenio_config_tuw/config.py b/invenio_config_tuw/config.py
index 899edbc..c94d4ea 100644
--- a/invenio_config_tuw/config.py
+++ b/invenio_config_tuw/config.py
@@ -8,7 +8,10 @@
"""Invenio module containing some customizations and configuration for TU Wien."""
from datetime import datetime
+from operator import attrgetter
+import invenio_base.app
+from importlib_metadata import entry_points as iter_entry_points
from invenio_app_rdm.config import (
NOTIFICATIONS_BACKENDS,
NOTIFICATIONS_BUILDERS,
@@ -389,3 +392,39 @@ NOTIFICATIONS_BACKENDS = {
}
NOTIFICATIONS_SETTINGS_VIEW_FUNCTION = notification_settings
+
+
+def sorted_app_loader(app, entry_points=None, modules=None):
+ """Application extension loader that operates in lexicographic order.
+
+ This is useful for us, to ensure that our `Invenio-Config-TUW` extension
+ is loaded before any of the others.
+ This enables us to hook into the startup process after the configuration
+ loader is done (and, especially, has finished interpreting environment
+ variables that we use heavily for configuration), but before any other
+ Invenio extensions have been loaded (and potentially started caching values).
+ """
+
+ def init_func(ext):
+ ext(app)
+
+ if entry_points:
+ for entry_point in entry_points:
+ unique_eps = set(iter_entry_points(group=entry_point))
+ for ep in sorted(unique_eps, key=attrgetter("name")):
+ try:
+ init_func(ep.load())
+ except Exception:
+ app.logger.error(f"Failed to initialize entry point: {ep}")
+ raise
+ if modules:
+ for m in modules:
+ try:
+ init_func(m)
+ except Exception:
+ app.logger.error(f"Failed to initialize module: {m}")
+ raise
+
+
+# override the app loader with our sorted variant
+invenio_base.app.app_loader = sorted_app_loader
diff --git a/invenio_config_tuw/ext.py b/invenio_config_tuw/ext.py
index 1f8d636..7ea1ad3 100644
--- a/invenio_config_tuw/ext.py
+++ b/invenio_config_tuw/ext.py
@@ -14,8 +14,12 @@ from flask_minify import Minify
from flask_security.signals import user_registered
from invenio_base.utils import obj_or_import_string
-from . import config
from .auth.utils import auto_trust_user
+from .startup.config import (
+ assemble_and_populate_config,
+ override_flask_config,
+ override_prefixed_config,
+)
@user_registered.connect
@@ -78,10 +82,16 @@ class InvenioConfigTUW(object):
return response
def init_config(self, app):
- """Initialize configuration."""
- for k in dir(config):
- if len(k.replace("_", "")) >= 3 and k.isupper():
- app.config.setdefault(k, getattr(config, k))
+ """Initialize configuration.
+
+ We use our comfortable position between the finalized initial loading of
+ configuration and the start of extension loading to perform a little bit
+ of magic on the configuration items, like building connection URIs from
+ their various pieces.
+ """
+ override_flask_config(app)
+ override_prefixed_config(app)
+ assemble_and_populate_config(app)
# the datacenter symbol seems to be the username for DataCite Fabrica
if app.config.get("DATACITE_ENABLED", False):
diff --git a/invenio_config_tuw/startup/__init__.py b/invenio_config_tuw/startup/__init__.py
index d25e9f3..8be7de3 100644
--- a/invenio_config_tuw/startup/__init__.py
+++ b/invenio_config_tuw/startup/__init__.py
@@ -14,7 +14,7 @@ initialized, and thus we can rely on them being already available.
"""
-from .config import finalize_config, patch_flask_create_url_adapter
+from .config import patch_flask_create_url_adapter
from .misc import (
customize_curation_request_type,
override_search_drafts_options,
@@ -24,7 +24,6 @@ from .misc import (
__all__ = (
"customize_curation_request_type",
- "finalize_config",
"override_search_drafts_options",
"patch_flask_create_url_adapter",
"register_menu_entries",
diff --git a/invenio_config_tuw/startup/config.py b/invenio_config_tuw/startup/config.py
index 90781d8..03aa16d 100644
--- a/invenio_config_tuw/startup/config.py
+++ b/invenio_config_tuw/startup/config.py
@@ -124,8 +124,10 @@ def assemble_broker_uri_from_parts(app):
else:
broker_url = "amqp://guest:guest@localhost:5672/"
- app.config.setdefault("BROKER_URL", broker_url)
- app.config.setdefault("CELERY_BROKER_URL", broker_url)
+ # celery doesn't like having BROKER_HOST *and* the other values set
+ app.config.pop("BROKER_HOST", None)
+ app.config["BROKER_URL"] = broker_url
+ app.config["CELERY_BROKER_URL"] = broker_url
def assemble_cache_uri_from_parts(app):
@@ -162,14 +164,15 @@ def assemble_cache_uri_from_parts(app):
ratelimit_storage_url = _make_redis_url(ratelimit_storage_db)
communities_identities_cache_url = _make_redis_url(communities_identities_db)
- app.config.setdefault("CACHE_TYPE", "redis")
- app.config.setdefault("CACHE_REDIS_URL", cache_redis_url)
- app.config.setdefault("IIIF_CACHE_REDIS_URL", cache_redis_url)
- app.config.setdefault("ACCOUNTS_SESSION_REDIS_URL", accounts_session_redis_url)
- app.config.setdefault("CELERY_RESULT_BACKEND", celery_results_backend_url)
- app.config.setdefault("RATELIMIT_STORAGE_URL", ratelimit_storage_url)
- app.config.setdefault(
- "COMMUNITIES_IDENTITIES_CACHE_REDIS_URL", communities_identities_cache_url
+ app.config["CACHE_TYPE"] = "redis"
+ app.config["CACHE_REDIS_URL"] = cache_redis_url
+ app.config["IIIF_CACHE_REDIS_URL"] = cache_redis_url
+ app.config["ACCOUNTS_SESSION_REDIS_URL"] = accounts_session_redis_url
+ app.config["CELERY_RESULT_BACKEND"] = celery_results_backend_url
+ app.config["RATELIMIT_STORAGE_URL"] = ratelimit_storage_url
+ app.config["RATELIMIT_STORAGE_URI"] = ratelimit_storage_url
+ app.config["COMMUNITIES_IDENTITIES_CACHE_REDIS_URL"] = (
+ communities_identities_cache_url
)
@@ -188,7 +191,7 @@ def assemble_site_urls_from_parts(app):
# note: 'invenio-cli run' likes to populate INVENIO_SITE_{UI,API}_URL...
app.config["SITE_UI_URL"] = LocalProxy(partial(_make_site_url, ""))
app.config["SITE_API_URL"] = LocalProxy(partial(_make_site_url, "/api"))
- app.config.setdefault("OAISERVER_ID_PREFIX", hostname)
+ app.config["OAISERVER_ID_PREFIX"] = hostname
def assemble_keycloak_config_from_parts(app):
@@ -201,8 +204,8 @@ def assemble_keycloak_config_from_parts(app):
"consumer_key": consumer_key,
"consumer_secret": consumer_secret,
}
- app.config.setdefault("OAUTHCLIENT_KEYCLOAK_APP_CREDENTIALS", app_credentials)
- app.config.setdefault("KEYCLOAK_APP_CREDENTIALS", app_credentials)
+ app.config["OAUTHCLIENT_KEYCLOAK_APP_CREDENTIALS"] = app_credentials
+ app.config["KEYCLOAK_APP_CREDENTIALS"] = app_credentials
base_url = app.config.get("OAUTHCLIENT_KEYCLOAK_BASE_URL")
realm = app.config.get("OAUTHCLIENT_KEYCLOAK_REALM")
@@ -226,10 +229,8 @@ def assemble_keycloak_config_from_parts(app):
**(app.config.get("OAUTHCLIENT_REMOTE_APPS") or {}),
"keycloak": remote_app,
}
- app.config.setdefault("OAUTHCLIENT_KEYCLOAK_REALM_URL", helper.realm_url)
- app.config.setdefault(
- "OAUTHCLIENT_KEYCLOAK_USER_INFO_URL", helper.user_info_url
- )
+ app.config["OAUTHCLIENT_KEYCLOAK_REALM_URL"] = helper.realm_url
+ app.config["OAUTHCLIENT_KEYCLOAK_USER_INFO_URL"] = helper.user_info_url
app.config["OAUTHCLIENT_REMOTE_APPS"] = remote_apps
@@ -291,21 +292,6 @@ def override_flask_config(app):
app.add_template_global(app.config, "config")
-def finalize_config(app):
- """Finalize the application configuration with a few tweaks.
-
- First, override the Flask configuration with the ``TUWConfig`` class.
- Then, use prefixed configuration items to override non-prefixed ones
- if a prefix is configured.
- Lastly, assemble some configuration items from their parts (like building
- the DB connection string from its pieces), and populate some unset
- values like salts.
- """
- override_flask_config(app)
- override_prefixed_config(app)
- assemble_and_populate_config(app)
-
-
def patch_flask_create_url_adapter(app):
"""Patch Flask's {host,subdomain}_matching with 3.1 behavior.
diff --git a/pyproject.toml b/pyproject.toml
index 6cb4e70..960d25b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,12 +66,18 @@ tests = [
[project.urls]
Repository = "https://gitlab.tuwien.ac.at/crdm/invenio-config-tuw"
-# entrypoints
+# Entrypoints
+# -----------
+# note: we omit the usual "invenio_*" prefix here, to be listed *before* the other
+# invenio extensions in lexicographic ordering so that we can piece together
+# the config items before they get picked up by the other extensions
+# since lexicographic ordering is currently not the default however, this
+# requires a hack performed in "invenio_config_tuw.config"
[project.entry-points."invenio_base.apps"]
-invenio_config_tuw = "invenio_config_tuw:InvenioConfigTUW"
+config_tuw = "invenio_config_tuw:InvenioConfigTUW"
[project.entry-points."invenio_base.api_apps"]
-invenio_config_tuw = "invenio_config_tuw:InvenioConfigTUW"
+config_tuw = "invenio_config_tuw:InvenioConfigTUW"
[project.entry-points."invenio_base.blueprints"]
invenio_config_tuw_settings = "invenio_config_tuw.users.views:user_settings_blueprint"
@@ -81,14 +87,12 @@ invenio_config_tuw_mail_handler = "invenio_config_tuw.startup:register_smtp_erro
invenio_config_tuw_search_drafts = "invenio_config_tuw.startup:override_search_drafts_options"
invenio_config_tuw_curation_settings = "invenio_config_tuw.startup:register_menu_entries"
invenio_config_tuw_curation_request = "invenio_config_tuw.startup:customize_curation_request_type"
-invenio_config_tuw_finalize_config = "invenio_config_tuw.startup:finalize_config"
invenio_config_tuw_patch_flask = "invenio_config_tuw.startup:patch_flask_create_url_adapter"
[project.entry-points."invenio_base.api_finalize_app"]
invenio_config_tuw_mail_handler = "invenio_config_tuw.startup:register_smtp_error_handler"
invenio_config_tuw_search_drafts = "invenio_config_tuw.startup:override_search_drafts_options"
invenio_config_tuw_curation_request = "invenio_config_tuw.startup:customize_curation_request_type"
-invenio_config_tuw_finalize_config = "invenio_config_tuw.startup:finalize_config"
invenio_config_tuw_patch_flask = "invenio_config_tuw.startup:patch_flask_create_url_adapter"
[project.entry-points."invenio_i18n.translations"]
@@ -106,7 +110,8 @@ invenio_config_tuw_curations = "invenio_config_tuw.curations.tasks"
# (which could be done in the ext.init_config(app) method)
zzz_invenio_config_tuw = "invenio_config_tuw.config"
-# configuration for tools
+# Configuration for tools
+# -----------------------
[tool.isort]
profile = "black"
--
GitLab
From 8bb8c1b62f65faa20b6256a2d792a5ecf0c2c6dd Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Tue, 11 Feb 2025 18:23:16 +0100
Subject: [PATCH 3/5] Update the name and comment for the config module
entrypoint
* `tuw_config` is a bit nicer than `zzz_invenio_config_tuw`, and should
still be the last entry in the list considering the usual naming
scheme
---
pyproject.toml | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 960d25b..5e15260 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -103,12 +103,13 @@ invenio_config_tuw = "invenio_config_tuw.tasks"
invenio_config_tuw_tiss = "invenio_config_tuw.tiss.tasks"
invenio_config_tuw_curations = "invenio_config_tuw.curations.tasks"
+# note: similar to above, we try to get listed *after* all the "invenio_*" entrypoints
+# so that we can override other packages' configuration items, but still get
+# executed before "invenio.cfg" (so that the values there take precedence)
+# unlike above, the lexicographic sorting is performed by `Invenio-Config`
+# out of the box
[project.entry-points."invenio_config.module"]
-# note: configs are loaded by `Invenio-Config` in lexicographic order of their
-# entrypoint names, and we want to override the other configs here
-# while not overriding `invenio.cfg`
-# (which could be done in the ext.init_config(app) method)
-zzz_invenio_config_tuw = "invenio_config_tuw.config"
+tuw_config = "invenio_config_tuw.config"
# Configuration for tools
# -----------------------
--
GitLab
From ba9102a72332c35cacbe6da404702a6e5435ac05 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Tue, 11 Feb 2025 18:37:12 +0100
Subject: [PATCH 4/5] Remove assertion about having the SECRET_KEY set
* Invenio-Config already takes care of checking the value in real
Invenio applications and issues a warning if it's unset
---
invenio_config_tuw/startup/config.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/invenio_config_tuw/startup/config.py b/invenio_config_tuw/startup/config.py
index 03aa16d..3f90323 100644
--- a/invenio_config_tuw/startup/config.py
+++ b/invenio_config_tuw/startup/config.py
@@ -237,9 +237,6 @@ def assemble_keycloak_config_from_parts(app):
def populate_unset_salt_values(app):
"""Populate the salt values if they're not set yet."""
secret_key = app.config.get("SECRET_KEY", None)
- if secret_key is None:
- raise RuntimeError("SECRET_KEY configuration is unset! Aborting.")
-
app.config.setdefault("CSRF_SECRET_SALT", secret_key)
app.config.setdefault("SECURITY_RESET_SALT", secret_key)
app.config.setdefault("SECURITY_LOGIN_SALT", secret_key)
--
GitLab
From 80cb9f6911f78d4ae0d23b0f3c6c55cb0100dfe2 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Tue, 11 Feb 2025 18:28:14 +0100
Subject: [PATCH 5/5] Bump version to v2025.1.6
---
CHANGES.rst | 8 ++++++++
invenio_config_tuw/__init__.py | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index d20dff2..6e2acd7 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -9,6 +9,14 @@ Changes
=======
+Version 2025.1.6 (released 2025-02-11)
+
+- Remove accidentally added MXID field from notification settings template
+- Move the configuration magic (assembly & overrides) to extension loading
+- Ensure that the `Invenio-Config-TUW` extension is loaded first
+- Rename the registered `invenio_config.module` entrypoint
+
+
Version 2025.1.5 (released 2025-02-10)
- Add background task for cleaning up "dead" files periodically
diff --git a/invenio_config_tuw/__init__.py b/invenio_config_tuw/__init__.py
index 012f95e..c4a33e0 100644
--- a/invenio_config_tuw/__init__.py
+++ b/invenio_config_tuw/__init__.py
@@ -9,6 +9,6 @@
from .ext import InvenioConfigTUW
-__version__ = "2025.1.5"
+__version__ = "2025.1.6"
__all__ = ("__version__", "InvenioConfigTUW")
--
GitLab