From 798907f187cdc2f80010960ac37f8ddd0a95bda2 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Tue, 11 Feb 2025 18:13:59 +0100
Subject: [PATCH] Perform config magic before extension loading
* previously, our configuration magic was performed as part of the
`finalize_app` entrypoint
* while this worked well enough for most use cases, some extensions
would break this model by caching values during their initialization
* especially caching-related extensions like to take eagerly create
clients with the current configuration items, which are hard to fix
afterwards
* keeping track of all the instantiated clients just to change their
connection details when we perform our magic really didn't seem like
a good approach
* so instead, we now made sure that our extension is loaded first and
can thus perform its configuration magic before any other extension
has a chance of caching values
* also, extension loading happens after the configuration loading is
done (especially the handling of environment variables)
---
invenio_config_tuw/config.py | 39 ++++++++++++++++++++
invenio_config_tuw/ext.py | 20 ++++++++---
invenio_config_tuw/startup/__init__.py | 3 +-
invenio_config_tuw/startup/config.py | 50 ++++++++++----------------
pyproject.toml | 17 +++++----
5 files changed, 84 insertions(+), 45 deletions(-)
diff --git a/invenio_config_tuw/config.py b/invenio_config_tuw/config.py
index 899edbc..c94d4ea 100644
--- a/invenio_config_tuw/config.py
+++ b/invenio_config_tuw/config.py
@@ -8,7 +8,10 @@
"""Invenio module containing some customizations and configuration for TU Wien."""
from datetime import datetime
+from operator import attrgetter
+import invenio_base.app
+from importlib_metadata import entry_points as iter_entry_points
from invenio_app_rdm.config import (
NOTIFICATIONS_BACKENDS,
NOTIFICATIONS_BUILDERS,
@@ -389,3 +392,39 @@ NOTIFICATIONS_BACKENDS = {
}
NOTIFICATIONS_SETTINGS_VIEW_FUNCTION = notification_settings
+
+
+def sorted_app_loader(app, entry_points=None, modules=None):
+ """Application extension loader that operates in lexicographic order.
+
+ This is useful for us, to ensure that our `Invenio-Config-TUW` extension
+ is loaded before any of the others.
+ This enables us to hook into the startup process after the configuration
+ loader is done (and, especially, has finished interpreting environment
+ variables that we use heavily for configuration), but before any other
+ Invenio extensions have been loaded (and potentially started caching values).
+ """
+
+ def init_func(ext):
+ ext(app)
+
+ if entry_points:
+ for entry_point in entry_points:
+ unique_eps = set(iter_entry_points(group=entry_point))
+ for ep in sorted(unique_eps, key=attrgetter("name")):
+ try:
+ init_func(ep.load())
+ except Exception:
+ app.logger.error(f"Failed to initialize entry point: {ep}")
+ raise
+ if modules:
+ for m in modules:
+ try:
+ init_func(m)
+ except Exception:
+ app.logger.error(f"Failed to initialize module: {m}")
+ raise
+
+
+# override the app loader with our sorted variant
+invenio_base.app.app_loader = sorted_app_loader
diff --git a/invenio_config_tuw/ext.py b/invenio_config_tuw/ext.py
index 1f8d636..7ea1ad3 100644
--- a/invenio_config_tuw/ext.py
+++ b/invenio_config_tuw/ext.py
@@ -14,8 +14,12 @@ from flask_minify import Minify
from flask_security.signals import user_registered
from invenio_base.utils import obj_or_import_string
-from . import config
from .auth.utils import auto_trust_user
+from .startup.config import (
+ assemble_and_populate_config,
+ override_flask_config,
+ override_prefixed_config,
+)
@user_registered.connect
@@ -78,10 +82,16 @@ class InvenioConfigTUW(object):
return response
def init_config(self, app):
- """Initialize configuration."""
- for k in dir(config):
- if len(k.replace("_", "")) >= 3 and k.isupper():
- app.config.setdefault(k, getattr(config, k))
+ """Initialize configuration.
+
+ We use our comfortable position between the finalized initial loading of
+ configuration and the start of extension loading to perform a little bit
+ of magic on the configuration items, like building connection URIs from
+ their various pieces.
+ """
+ override_flask_config(app)
+ override_prefixed_config(app)
+ assemble_and_populate_config(app)
# the datacenter symbol seems to be the username for DataCite Fabrica
if app.config.get("DATACITE_ENABLED", False):
diff --git a/invenio_config_tuw/startup/__init__.py b/invenio_config_tuw/startup/__init__.py
index d25e9f3..8be7de3 100644
--- a/invenio_config_tuw/startup/__init__.py
+++ b/invenio_config_tuw/startup/__init__.py
@@ -14,7 +14,7 @@ initialized, and thus we can rely on them being already available.
"""
-from .config import finalize_config, patch_flask_create_url_adapter
+from .config import patch_flask_create_url_adapter
from .misc import (
customize_curation_request_type,
override_search_drafts_options,
@@ -24,7 +24,6 @@ from .misc import (
__all__ = (
"customize_curation_request_type",
- "finalize_config",
"override_search_drafts_options",
"patch_flask_create_url_adapter",
"register_menu_entries",
diff --git a/invenio_config_tuw/startup/config.py b/invenio_config_tuw/startup/config.py
index 90781d8..03aa16d 100644
--- a/invenio_config_tuw/startup/config.py
+++ b/invenio_config_tuw/startup/config.py
@@ -124,8 +124,10 @@ def assemble_broker_uri_from_parts(app):
else:
broker_url = "amqp://guest:guest@localhost:5672/"
- app.config.setdefault("BROKER_URL", broker_url)
- app.config.setdefault("CELERY_BROKER_URL", broker_url)
+ # celery doesn't like having BROKER_HOST *and* the other values set
+ app.config.pop("BROKER_HOST", None)
+ app.config["BROKER_URL"] = broker_url
+ app.config["CELERY_BROKER_URL"] = broker_url
def assemble_cache_uri_from_parts(app):
@@ -162,14 +164,15 @@ def assemble_cache_uri_from_parts(app):
ratelimit_storage_url = _make_redis_url(ratelimit_storage_db)
communities_identities_cache_url = _make_redis_url(communities_identities_db)
- app.config.setdefault("CACHE_TYPE", "redis")
- app.config.setdefault("CACHE_REDIS_URL", cache_redis_url)
- app.config.setdefault("IIIF_CACHE_REDIS_URL", cache_redis_url)
- app.config.setdefault("ACCOUNTS_SESSION_REDIS_URL", accounts_session_redis_url)
- app.config.setdefault("CELERY_RESULT_BACKEND", celery_results_backend_url)
- app.config.setdefault("RATELIMIT_STORAGE_URL", ratelimit_storage_url)
- app.config.setdefault(
- "COMMUNITIES_IDENTITIES_CACHE_REDIS_URL", communities_identities_cache_url
+ app.config["CACHE_TYPE"] = "redis"
+ app.config["CACHE_REDIS_URL"] = cache_redis_url
+ app.config["IIIF_CACHE_REDIS_URL"] = cache_redis_url
+ app.config["ACCOUNTS_SESSION_REDIS_URL"] = accounts_session_redis_url
+ app.config["CELERY_RESULT_BACKEND"] = celery_results_backend_url
+ app.config["RATELIMIT_STORAGE_URL"] = ratelimit_storage_url
+ app.config["RATELIMIT_STORAGE_URI"] = ratelimit_storage_url
+ app.config["COMMUNITIES_IDENTITIES_CACHE_REDIS_URL"] = (
+ communities_identities_cache_url
)
@@ -188,7 +191,7 @@ def assemble_site_urls_from_parts(app):
# note: 'invenio-cli run' likes to populate INVENIO_SITE_{UI,API}_URL...
app.config["SITE_UI_URL"] = LocalProxy(partial(_make_site_url, ""))
app.config["SITE_API_URL"] = LocalProxy(partial(_make_site_url, "/api"))
- app.config.setdefault("OAISERVER_ID_PREFIX", hostname)
+ app.config["OAISERVER_ID_PREFIX"] = hostname
def assemble_keycloak_config_from_parts(app):
@@ -201,8 +204,8 @@ def assemble_keycloak_config_from_parts(app):
"consumer_key": consumer_key,
"consumer_secret": consumer_secret,
}
- app.config.setdefault("OAUTHCLIENT_KEYCLOAK_APP_CREDENTIALS", app_credentials)
- app.config.setdefault("KEYCLOAK_APP_CREDENTIALS", app_credentials)
+ app.config["OAUTHCLIENT_KEYCLOAK_APP_CREDENTIALS"] = app_credentials
+ app.config["KEYCLOAK_APP_CREDENTIALS"] = app_credentials
base_url = app.config.get("OAUTHCLIENT_KEYCLOAK_BASE_URL")
realm = app.config.get("OAUTHCLIENT_KEYCLOAK_REALM")
@@ -226,10 +229,8 @@ def assemble_keycloak_config_from_parts(app):
**(app.config.get("OAUTHCLIENT_REMOTE_APPS") or {}),
"keycloak": remote_app,
}
- app.config.setdefault("OAUTHCLIENT_KEYCLOAK_REALM_URL", helper.realm_url)
- app.config.setdefault(
- "OAUTHCLIENT_KEYCLOAK_USER_INFO_URL", helper.user_info_url
- )
+ app.config["OAUTHCLIENT_KEYCLOAK_REALM_URL"] = helper.realm_url
+ app.config["OAUTHCLIENT_KEYCLOAK_USER_INFO_URL"] = helper.user_info_url
app.config["OAUTHCLIENT_REMOTE_APPS"] = remote_apps
@@ -291,21 +292,6 @@ def override_flask_config(app):
app.add_template_global(app.config, "config")
-def finalize_config(app):
- """Finalize the application configuration with a few tweaks.
-
- First, override the Flask configuration with the ``TUWConfig`` class.
- Then, use prefixed configuration items to override non-prefixed ones
- if a prefix is configured.
- Lastly, assemble some configuration items from their parts (like building
- the DB connection string from its pieces), and populate some unset
- values like salts.
- """
- override_flask_config(app)
- override_prefixed_config(app)
- assemble_and_populate_config(app)
-
-
def patch_flask_create_url_adapter(app):
"""Patch Flask's {host,subdomain}_matching with 3.1 behavior.
diff --git a/pyproject.toml b/pyproject.toml
index 6cb4e70..960d25b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,12 +66,18 @@ tests = [
[project.urls]
Repository = "https://gitlab.tuwien.ac.at/crdm/invenio-config-tuw"
-# entrypoints
+# Entrypoints
+# -----------
+# note: we omit the usual "invenio_*" prefix here, to be listed *before* the other
+# invenio extensions in lexicographic ordering so that we can piece together
+# the config items before they get picked up by the other extensions
+# since lexicographic ordering is currently not the default however, this
+# requires a hack performed in "invenio_config_tuw.config"
[project.entry-points."invenio_base.apps"]
-invenio_config_tuw = "invenio_config_tuw:InvenioConfigTUW"
+config_tuw = "invenio_config_tuw:InvenioConfigTUW"
[project.entry-points."invenio_base.api_apps"]
-invenio_config_tuw = "invenio_config_tuw:InvenioConfigTUW"
+config_tuw = "invenio_config_tuw:InvenioConfigTUW"
[project.entry-points."invenio_base.blueprints"]
invenio_config_tuw_settings = "invenio_config_tuw.users.views:user_settings_blueprint"
@@ -81,14 +87,12 @@ invenio_config_tuw_mail_handler = "invenio_config_tuw.startup:register_smtp_erro
invenio_config_tuw_search_drafts = "invenio_config_tuw.startup:override_search_drafts_options"
invenio_config_tuw_curation_settings = "invenio_config_tuw.startup:register_menu_entries"
invenio_config_tuw_curation_request = "invenio_config_tuw.startup:customize_curation_request_type"
-invenio_config_tuw_finalize_config = "invenio_config_tuw.startup:finalize_config"
invenio_config_tuw_patch_flask = "invenio_config_tuw.startup:patch_flask_create_url_adapter"
[project.entry-points."invenio_base.api_finalize_app"]
invenio_config_tuw_mail_handler = "invenio_config_tuw.startup:register_smtp_error_handler"
invenio_config_tuw_search_drafts = "invenio_config_tuw.startup:override_search_drafts_options"
invenio_config_tuw_curation_request = "invenio_config_tuw.startup:customize_curation_request_type"
-invenio_config_tuw_finalize_config = "invenio_config_tuw.startup:finalize_config"
invenio_config_tuw_patch_flask = "invenio_config_tuw.startup:patch_flask_create_url_adapter"
[project.entry-points."invenio_i18n.translations"]
@@ -106,7 +110,8 @@ invenio_config_tuw_curations = "invenio_config_tuw.curations.tasks"
# (which could be done in the ext.init_config(app) method)
zzz_invenio_config_tuw = "invenio_config_tuw.config"
-# configuration for tools
+# Configuration for tools
+# -----------------------
[tool.isort]
profile = "black"
--
GitLab