From 06dedb59850c0845fcd885e63c90ef7510294b3a Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Thu, 23 Jan 2025 11:31:45 +0100
Subject: [PATCH] Implement curation-related celery tasks
* automatic accept of submissions for the test system
* reminder for uploaders about accepted but still unpublished records
* reminder for reviewers about open curation requests
---
invenio_config_tuw/config.py | 18 ++
invenio_config_tuw/ext.py | 23 +++
invenio_config_tuw/notifications.py | 34 +++-
invenio_config_tuw/tasks.py | 283 +++++++++++++++++++++++++++-
4 files changed, 355 insertions(+), 3 deletions(-)
diff --git a/invenio_config_tuw/config.py b/invenio_config_tuw/config.py
index e635cbd..b1ae213 100644
--- a/invenio_config_tuw/config.py
+++ b/invenio_config_tuw/config.py
@@ -33,6 +33,7 @@ from .permissions import (
TUWRequestsPermissionPolicy,
)
from .services import TUWRecordsComponents
+from .tasks import auto_generate_curation_request_remarks
from .users import (
TUWUserPreferencesSchema,
TUWUserProfileSchema,
@@ -78,6 +79,23 @@ If set to `None`, the first available option will be used:
* the first entry for `APP_ALLOWED_HOSTS`
"""
+CONFIG_TUW_AUTO_ACCEPT_CURATION_REQUESTS = False
+"""Whether or not the system should auto-accept curation requests.
+
+This can be either a boolean value to be returned for all requests, or it can be
+a function that takes the request as argument and returns a boolean value.
+Functions can be either supplied via reference, or via import string.
+"""
+
+CONFIG_TUW_AUTO_COMMENT_CURATION_REQUESTS = auto_generate_curation_request_remarks
+"""A function to automatically generate remarks for record curation requests.
+
+The function must take the request as argument and return a list of messages (strings)
+to be used to create a system comment on the request.
+Functions can be either supplied via reference, or via import string.
+A value of ``None`` disables this feature.
+"""
+
# Invenio-Mail
# ============
diff --git a/invenio_config_tuw/ext.py b/invenio_config_tuw/ext.py
index c7e2805..d21bbc9 100644
--- a/invenio_config_tuw/ext.py
+++ b/invenio_config_tuw/ext.py
@@ -7,10 +7,13 @@
"""Invenio module containing some customizations and configuration for TU Wien."""
+from typing import List
+
from flask import current_app
from flask.config import Config
from flask_minify import Minify
from flask_security.signals import user_registered
+from invenio_base.utils import obj_or_import_string
from . import config
from .auth.utils import auto_trust_user
@@ -92,6 +95,26 @@ class InvenioConfigTUW(object):
minify = Minify(app, static=False, go=False)
app.extensions["flask-minify"] = minify
+ def auto_accept_record_curation_request(self, request) -> bool:
+ """Check if the request should be auto-accepted according to the config."""
+ auto_accept = current_app.config.get(
+ "CONFIG_TUW_AUTO_ACCEPT_CURATION_REQUESTS", False
+ )
+ if isinstance(auto_accept, bool):
+ return auto_accept
+
+ return obj_or_import_string(auto_accept)(request)
+
+ def generate_record_curation_request_remarks(self, request) -> List[str]:
+ """Generate remarks to automatically add as comment to the curation request."""
+ generate_remarks = current_app.config.get(
+ "CONFIG_TUW_AUTO_COMMENT_CURATION_REQUESTS", None
+ )
+ if generate_remarks is None:
+ return []
+
+ return obj_or_import_string(generate_remarks)(request)
+
@property
def curations_enabled(self):
"""Shorthand for ``current_app.config.get["CONFIG_TUW_CURATIONS_ENABLED"]``."""
diff --git a/invenio_config_tuw/notifications.py b/invenio_config_tuw/notifications.py
index 5bbd7b9..24d2066 100644
--- a/invenio_config_tuw/notifications.py
+++ b/invenio_config_tuw/notifications.py
@@ -187,7 +187,7 @@ class TUWCurationRequestUploaderResubmitNotificationBuilder(
class TUWCurationResubmitAction(CurationResubmitAction):
- """Notify both uploader and reviewer on resubmit."""
+ """Notify both uploader and reviewer on resubmit, and auto-review."""
def execute(self, identity, uow):
"""Notify uploader when the record gets resubmitted for review."""
@@ -198,6 +198,36 @@ class TUWCurationResubmitAction(CurationResubmitAction):
)
)
)
+ uow.register(
+ TUWTaskOp(auto_review_curation_request, str(self.request.id), countdown=15)
+ )
+ return super().execute(identity, uow)
+
+
+class TUWCurationSubmitAction(CurationSubmitAction):
+ """Submit action with a hook for automatic reviews.
+
+ Note: It looks like this isn't really being used, in favor of "create & submit".
+ """
+
+ def execute(self, identity, uow):
+ """Register auto-review task and perform the submit action."""
+ uow.register(
+ TUWTaskOp(auto_review_curation_request, str(self.request.id), countdown=15)
+ )
+
+ return super().execute(identity, uow)
+
+
+class TUWCurationCreateAndSubmitAction(CurationCreateAndSubmitAction):
+ """'Create & submit' action with a hook for automatic reviews."""
+
+ def execute(self, identity, uow):
+ """Register auto-review task and perform the 'create & submit' action."""
+ uow.register(
+ TUWTaskOp(auto_review_curation_request, str(self.request.id), countdown=15)
+ )
+
return super().execute(identity, uow)
@@ -206,5 +236,7 @@ class TUWCurationRequest(CurationRequest):
available_actions = {
**CurationRequest.available_actions,
+ "create": TUWCurationCreateAndSubmitAction,
+ "submit": TUWCurationSubmitAction,
"resubmit": TUWCurationResubmitAction,
}
diff --git a/invenio_config_tuw/tasks.py b/invenio_config_tuw/tasks.py
index a9dad2d..a3fa832 100644
--- a/invenio_config_tuw/tasks.py
+++ b/invenio_config_tuw/tasks.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2023 TU Wien.
+# Copyright (C) 2023-2025 TU Wien.
#
# Invenio-Config-TUW is free software; you can redistribute it and/or modify
# it under the terms of the MIT License; see LICENSE file for more details.
@@ -9,20 +9,27 @@
import copy
from collections import defaultdict
+from datetime import UTC, date, datetime, timedelta
from difflib import SequenceMatcher
from typing import List, Optional
import requests
from celery import shared_task
-from flask import current_app, render_template
+from celery.schedules import crontab
+from flask import current_app, render_template, url_for
from invenio_access.permissions import system_identity
from invenio_accounts.proxies import current_datastore
from invenio_db import db
from invenio_mail.tasks import send_email
+from invenio_notifications.tasks import broadcast_notification
from invenio_rdm_records.proxies import current_rdm_records_service as records_service
from invenio_records_resources.services.uow import RecordIndexOp, UnitOfWork
+from invenio_requests.customizations.event_types import CommentEventType
+from invenio_requests.proxies import current_events_service as events_service
+from invenio_requests.proxies import current_requests_service as requests_service
from invenio_vocabularies.contrib.names.api import Name
+from .proxies import current_config_tuw
from .tiss import Employee, fetch_tiss_data
@@ -107,6 +114,53 @@ def _calc_name_distance(
return fn_dist + ln_dist
+def auto_generate_curation_request_remarks(request):
+ """Auto-generate remarks on the curation request based on simple rules."""
+ record = request.topic.resolve()
+ remarks = []
+
+ # check if the description has been edited
+ deposit_form_defaults = current_app.config.get("APP_RDM_DEPOSIT_FORM_DEFAULTS", {})
+ default_description = deposit_form_defaults.get("description", None)
+ if callable(default_description):
+ default_description = default_description()
+
+ description = record.metadata["description"] or ""
+ if description == default_description:
+ remarks.append("The description is still the default template, please edit.")
+ elif "to be edited" in description.lower():
+ remarks.append(
+ "The description looks like it's meant to still be edited, please check."
+ )
+
+ # check if a license has been applied
+ if not record.metadata.get("rights", []):
+ remarks.append(
+ "Not assigning a license strongly restricts the legal reusability (all rights are reserved). Is this intentional?"
+ )
+
+ return remarks
+
+
+def _get_last_request_action_timestamp(request):
+ """Get the timestamp of the last log event on the request, or its creation time."""
+ # check if the request has been sitting around for a while
+ events = events_service.search(
+ identity=system_identity,
+ request_id=request["id"],
+ )
+ log_events = [e for e in events if e["type"] == "L"]
+ if not log_events:
+ # curation requests without any log events are in "submitted" state,
+ # and we need to look at the creation/update time
+ timestamp = datetime.fromisoformat(request["created"])
+ else:
+ # otherwise, we look at the last log event
+ timestamp = datetime.fromisoformat(log_events[-1]["created"])
+
+ return timestamp
+
+
@shared_task(ignore_result=True)
def sync_names_from_tiss() -> dict:
"""Look up TU Wien employees via TISS and update the names vocabulary."""
@@ -238,3 +292,228 @@ def send_publication_notification_email(recid: str, user_id: Optional[str] = Non
"recipients": [user.email],
}
)
+
+
+@shared_task(ignore_result=True)
+def send_acceptance_reminder_to_uploader(recid: str):
+ """Send a reminder notification about the accepted review to the uploader."""
+ from .notifications import UserNotificationBuilder
+
+ draft = records_service.read_draft(identity=system_identity, id_=recid)._obj
+ if (owner := draft.parent.access.owned_by) is None:
+ return
+ else:
+ owner = owner.resolve()
+
+ # NOTE: this requires the UI app, which is the base for the celery app
+ deposit_form_url = url_for(
+ "invenio_app_rdm_records.deposit_edit",
+ pid_value=draft.pid.pid_value,
+ _external=True,
+ )
+ title = draft.metadata["title"]
+ message = (
+ f'Reminder: Your record "{title}" has been reviewed and is ready for publication.\n'
+ f"You can publish it on the deposit form: {deposit_form_url}"
+ )
+ html_message = (
+ f'Reminder: Your record "{title}" has been reviewed and is ready for publication.<br />'
+ f'You can publish it on the <a href="{deposit_form_url}">deposit form</a>.'
+ )
+
+ notification = UserNotificationBuilder.build(
+ receiver={"user": owner.id},
+ subject="ℹ️ Reminder: Your record is ready for publication!",
+ message=message,
+ html_message=html_message,
+ )
+ broadcast_notification(notification.dumps())
+
+
+@shared_task(ignore_result=True)
+def send_open_requests_reminder_to_reviewers(request_ids: List[str]):
+ """Send a reminder notification about open curation requests to the reviewers."""
+ from .notifications import GroupNotificationBuilder
+
+ reviewer_role_name = current_app.config["CURATIONS_MODERATION_ROLE"]
+ plain_lines, html_lines = [], []
+ for reqid in request_ids:
+ # we assume that only a single request has the same UUID
+ request = list(requests_service.search(system_identity, q=f"uuid:{reqid}"))[0]
+ title = request["title"]
+
+ # NOTE: the reported "self_html" URL is currently broken
+ # (points to "/requests/..." rather than "/me/requests/...")
+ request_url = url_for(
+ "invenio_app_rdm_requests.user_dashboard_request_view",
+ request_pid_value=request["id"],
+ _external=True,
+ )
+
+ plain_lines.append(f'* "{title}": {request_url}')
+ html_lines.append(f'<li>"<a href="{request_url}">{title}</a>"</li>')
+
+ plain_list = "\n".join(plain_lines)
+ message = f"Reminder: Please review the following requests, they've been waiting for a response for a while:\n{plain_list}"
+ html_list = "<ul>" + "".join(html_lines) + "</ul>"
+ html_message = f"<p>Reminder: Please review the following requests, they've been waiting for a response for a while:</p><p>{html_list}</p>"
+ notification = GroupNotificationBuilder.build(
+ receiver={"group": reviewer_role_name},
+ subject="⚠️ Reminder: There are some open curation requests",
+ message=message,
+ html_message=html_message,
+ )
+ broadcast_notification(notification.dumps())
+
+
+@shared_task(ignore_result=True)
+def remind_uploaders_about_accepted_reviews(
+ remind_after_days: Optional[List[int]] = None,
+):
+ """Find curation reviews that were accepted a while ago and remind the uploaders.
+
+ ``remind_after_days`` specifies after how many days of inactivity reminders
+ should be sent out to reviewers.
+ Default: ``[1, 3, 5, 7, 10, 14, 30]``
+ """
+ if remind_after_days is None:
+ remind_after_days = [1, 3, 5, 7, 10, 14, 30]
+
+ # first, we get a list of all requests that have been updated in the last year
+ # but excluding today (with the brackets "[ ... }")
+ #
+ # note: the date query is intended to set a soft limit on the number of results
+ # to avoid unbounded degradation over time
+ # also, we don't expect any requests that haven't been updated in over a year
+ # to still be relevant for notifications
+ #
+ # note: querying for "L"-type "accepted" events won't work, as the information
+ # about the action is stored in the payload which is disabled for indexing
+ # as of InvenioRDM v12
+ start_date = (date.today() - timedelta(days=365)).isoformat()
+ today = date.today().isoformat()
+ accepted_curation_requests = requests_service.search(
+ identity=system_identity,
+ q=(
+ "type:rdm-curation AND "
+ "status:accepted AND "
+ f"updated:[{start_date} TO {today}}}"
+ ),
+ )
+
+ now = datetime.now(tz=UTC)
+ for request in accepted_curation_requests:
+ if isinstance(request, dict):
+ # we don't want dictionaries, we want request API classes
+ # BEWARE: other than for resolving the topic, this is useless!
+ request = requests_service.record_cls(request)
+
+ record = request.topic.resolve()
+ if record.is_published:
+ continue
+
+ # check if we're hitting one of the reminder dates
+ timestamp = _get_last_request_action_timestamp(request)
+ if abs((now - timestamp).days) in remind_after_days:
+ send_acceptance_reminder_to_uploader.delay(record.pid.pid_value)
+
+
+@shared_task(ignore_result=True)
+def remind_reviewers_about_open_reviews(remind_after_days: Optional[List[int]] = None):
+ """Remind a user about having an accepted review for an unpublished record.
+
+ ``remind_after_days`` specifies after how many days of inactivity reminders
+ should be sent out to reviewers.
+ Default: ``[1, 3, 5, 7, 10, 14, 30]``
+ """
+ if remind_after_days is None:
+ remind_after_days = [1, 3, 5, 7, 10, 14, 30]
+
+ # note: we don't expect a lot of results for this query at any time,
+ # as the number of open requests should be low at any point
+ open_curation_requests = requests_service.search(
+ identity=system_identity,
+ q="type:rdm-curation AND (status:submitted OR status:resubmitted)",
+ )
+
+ now = datetime.now(tz=UTC)
+ stale_request_ids = []
+ for request in open_curation_requests:
+ if isinstance(request, dict):
+ # we don't want dictionaries, we want request API classes
+ # BEWARE: other than for resolving the topic, this is useless!
+ request = requests_service.record_cls(request)
+
+ # quick sanity check: don't notify about weird zombie requests
+ record = request.topic.resolve()
+ if record.is_published:
+ continue
+
+ # check if we're hitting one of the reminder dates
+ timestamp = _get_last_request_action_timestamp(request)
+ if abs((now - timestamp).days) in remind_after_days:
+ stale_request_ids.append(request["id"])
+
+ if stale_request_ids:
+ send_open_requests_reminder_to_reviewers.delay(stale_request_ids)
+
+
+@shared_task(ignore_result=True)
+def auto_review_curation_request(request_id: str):
+ """Have the system automatically accept a submission request."""
+ request = requests_service.read(id_=request_id, identity=system_identity)._obj
+ if request.status not in ["submitted", "resubmitted"]:
+ return
+
+ # if configured, let the system automatically start a review and accept
+ auto_accept = current_config_tuw.auto_accept_record_curation_request(request)
+ if auto_accept:
+ requests_service.execute_action(
+ identity=system_identity,
+ id_=request_id,
+ action="review",
+ )
+
+ # auto-generate a mini review about the record
+ remarks = current_config_tuw.generate_record_curation_request_remarks(request)
+ if remarks:
+ events_service.create(
+ identity=system_identity,
+ request_id=request_id,
+ event_type=CommentEventType,
+ data={
+ "payload": {
+ "content": "\n".join([f"<p>{remark}</p>" for remark in remarks]),
+ "format": "html",
+ }
+ },
+ )
+
+ if auto_accept:
+ requests_service.execute_action(
+ identity=system_identity,
+ id_=request_id,
+ action="accept",
+ data={
+ "payload": {
+ "content": "<p>Automatically accepted by the system</p>",
+ "format": "html",
+ }
+ },
+ )
+
+
+CELERY_BEAT_SCHEDULE = {
+ "tiss-name-sync": {
+ "task": "invenio_config_tuw.tasks.sync_names_from_tiss",
+ "schedule": crontab(minute=0, hour=3, day_of_week="sat"),
+ },
+ "reviewers-open-requests-reminder": {
+ "task": "invenio_config_tuw.tasks.remind_reviewers_about_open_reviews",
+ "schedule": crontab(minute=30, hour=8),
+ },
+ "uploaders-acceptance-reminder": {
+ "task": "invenio_config_tuw.tasks.remind_uploaders_about_accepted_reviews",
+ "schedule": crontab(minute=30, hour=8),
+ },
+}
--
GitLab