From fdfe19d768cbe6611a5b61dc5efb7a24233a8573 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Fri, 22 Jan 2021 16:02:17 +0100
Subject: [PATCH] Adjust module for December release and change too many things
at once
* apply Black formatting
* use plural for sub-commands, for consistency
* fix convert_to_recid() to always fetch the first PID of type 'recid'
* add command to reindex all (or some) records
* update method for instantiating RecordServices: allow the configuration
of a factory function that creates a RecordService as desired
(default: fetches services from current_rdm_records)
---
invenio_utilities_tuw/cli/cli.py | 8 +--
.../cli/{draft.py => drafts.py} | 39 +++++-----
invenio_utilities_tuw/cli/files.py | 24 ++++---
.../cli/{record.py => records.py} | 71 ++++++++++++++-----
invenio_utilities_tuw/cli/utils.py | 27 ++++---
invenio_utilities_tuw/config.py | 20 ++++--
invenio_utilities_tuw/utils.py | 51 +++++++++++++
7 files changed, 173 insertions(+), 67 deletions(-)
rename invenio_utilities_tuw/cli/{draft.py => drafts.py} (92%)
rename invenio_utilities_tuw/cli/{record.py => records.py} (66%)
create mode 100644 invenio_utilities_tuw/utils.py
diff --git a/invenio_utilities_tuw/cli/cli.py b/invenio_utilities_tuw/cli/cli.py
index 283fd6f..cfe3a1c 100644
--- a/invenio_utilities_tuw/cli/cli.py
+++ b/invenio_utilities_tuw/cli/cli.py
@@ -2,9 +2,9 @@
import click
-from .draft import draft
+from .drafts import drafts
from .files import files
-from .record import record
+from .records import records
from .users import users
@@ -14,7 +14,7 @@ def utilities():
pass
-utilities.add_command(draft)
+utilities.add_command(drafts)
utilities.add_command(files)
-utilities.add_command(record)
+utilities.add_command(records)
utilities.add_command(users)
diff --git a/invenio_utilities_tuw/cli/draft.py b/invenio_utilities_tuw/cli/drafts.py
similarity index 92%
rename from invenio_utilities_tuw/cli/draft.py
rename to invenio_utilities_tuw/cli/drafts.py
index d0e625b..738ea61 100644
--- a/invenio_utilities_tuw/cli/draft.py
+++ b/invenio_utilities_tuw/cli/drafts.py
@@ -9,13 +9,8 @@ import click
from flask.cli import with_appcontext
from invenio_files_rest.models import ObjectVersion
from invenio_rdm_records.records.models import DraftMetadata
-from invenio_rdm_records.services.services import (
- BibliographicDraftFilesService as DraftFileService,
-)
-from invenio_rdm_records.services.services import (
- BibliographicRecordService as RecordService,
-)
+from ..utils import get_draft_file_service, get_record_service
from .utils import (
convert_to_recid,
create_record_from_metadata,
@@ -51,18 +46,18 @@ option_pid_value = click.option(
@click.group()
-def draft():
+def drafts():
"""Utility commands for creation and publication of drafts."""
pass
-@draft.command("list")
+@drafts.command("list")
@option_as_user
@with_appcontext
def list_drafts(user):
"""List all drafts accessible to the given user."""
identity = get_identity_for_user(user)
- service = RecordService()
+ service = get_record_service()
recids = [
dm.json["id"]
for dm in DraftMetadata.query.all()
@@ -79,7 +74,7 @@ def list_drafts(user):
pass
-@draft.command("create")
+@drafts.command("create")
@click.argument("metadata_path", type=click.Path(exists=True))
@option_as_user
@click.option(
@@ -127,7 +122,7 @@ def create_draft(metadata_path, publish, user):
msg = "ignored in '{}': {}".format(deposit_files_path, ignored)
click.secho(msg, fg="red", err=True)
- service = DraftFileService()
+ service = get_draft_file_service()
service.init_files(
id_=recid, identity=identity, data=[{"key": fn} for fn in file_names]
)
@@ -144,13 +139,13 @@ def create_draft(metadata_path, publish, user):
raise Exception("neither a file nor a directory: %s" % metadata_path)
if publish:
- service = RecordService()
+ service = get_record_service()
service.publish(id_=recid, identity=identity)
click.secho(recid, fg="green")
-@draft.command("update")
+@drafts.command("update")
@click.argument("metadata_file", type=click.File("r"))
@option_pid_value
@option_pid_type
@@ -166,7 +161,7 @@ def update_draft(metadata_file, pid, pid_type, user, patch):
"""Update the specified draft's metadata."""
pid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = RecordService()
+ service = get_record_service()
metadata = json.load(metadata_file)
if patch:
@@ -177,7 +172,7 @@ def update_draft(metadata_file, pid, pid_type, user, patch):
click.secho(pid, fg="green")
-@draft.command("publish")
+@drafts.command("publish")
@option_pid_value
@option_pid_type
@option_as_user
@@ -186,12 +181,12 @@ def publish_draft(pid, pid_type, user):
"""Publish the specified draft."""
pid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = RecordService()
+ service = get_record_service()
service.publish(id_=pid, identity=identity)
click.secho(pid, fg="green")
-@draft.command("delete")
+@drafts.command("delete")
@option_pid_value
@option_pid_type
@option_as_user
@@ -200,12 +195,12 @@ def delete_draft(pid, pid_type, user):
"""Delete the specified draft."""
pid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = RecordService()
+ service = get_record_service()
service.delete_draft(id_=pid, identity=identity)
click.secho(pid, fg="red")
-@draft.group()
+@drafts.group()
def files():
"""Manage files deposited with the draft."""
pass
@@ -221,7 +216,7 @@ def add_files(filepaths, pid, pid_type, user):
"""Add the specified files to the draft."""
recid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = DraftFileService()
+ service = get_draft_file_service()
paths = []
for file_path in filepaths:
@@ -271,7 +266,7 @@ def remove_files(filekeys, pid, pid_type, user):
"""Remove the deposited files."""
recid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = DraftFileService()
+ service = get_draft_file_service()
for file_key in filekeys:
service.delete_file(id_=recid, file_key=file_key, identity=identity)
@@ -288,7 +283,7 @@ def list_files(pid, pid_type, user):
"""Show a list of files deposited with the draft."""
recid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = DraftFileService()
+ service = get_draft_file_service()
file_results = service.list_files(id_=recid, identity=identity)
for f in file_results.entries:
ov = ObjectVersion.get(f["bucket_id"], f["key"], f["version_id"])
diff --git a/invenio_utilities_tuw/cli/files.py b/invenio_utilities_tuw/cli/files.py
index 9db2438..242f301 100644
--- a/invenio_utilities_tuw/cli/files.py
+++ b/invenio_utilities_tuw/cli/files.py
@@ -5,15 +5,12 @@ from collections import defaultdict
import click
from flask.cli import with_appcontext
from invenio_db import db
-from invenio_files_rest.models import Bucket, ObjectVersion, FileInstance
-from invenio_rdm_records.records.models import RecordMetadata, DraftMetadata
-from invenio_rdm_records.services.services import (
- BibliographicRecordService as RecordService,
-)
+from invenio_files_rest.models import Bucket, FileInstance, ObjectVersion
+from invenio_rdm_records.records.models import DraftMetadata, RecordMetadata
+from ..utils import get_record_service
from .utils import convert_to_recid, get_identity_for_user
-
option_as_user = click.option(
"--as-user",
"-u",
@@ -64,7 +61,7 @@ def list_deleted_files(user, pid, pid_type):
(via its PID).
"""
recid = convert_to_recid(pid, pid_type) if pid else None
- service = RecordService()
+ service = get_record_service()
identity = get_identity_for_user(user)
# if a PID was specified, limit the cleaning to this record's bucket
@@ -106,7 +103,7 @@ def hard_delete_files(user, pid, pid_type):
(via its PID).
"""
recid = convert_to_recid(pid, pid_type) if pid else None
- service = RecordService()
+ service = get_record_service()
identity = get_identity_for_user(user)
# if a PID was specified, limit the cleaning to this record's bucket
@@ -152,7 +149,13 @@ def list_orphan_files():
"""List files that aren't referenced in any records (anymore)."""
# TODO iterate over all records & drafts, get their buckets
# and check which buckets from the db aren't listed
- bucket_ids = set((r.bucket.id for r in (RecordMetadata.query.all() + DraftMetadata.query.all()) if r.bucket is not None))
+ bucket_ids = set(
+ (
+ r.bucket.id
+ for r in (RecordMetadata.query.all() + DraftMetadata.query.all())
+ if r.bucket is not None
+ )
+ )
print(len(bucket_ids))
buckets = Bucket.query.filter(~Bucket.id.in_(bucket_ids)).all()
print(len(buckets))
@@ -171,9 +174,8 @@ def list_orphan_files():
@with_appcontext
def clean_files(user):
"""Remove files that do not have associated ObjectVersions (anymore)."""
- service = RecordService()
+ service = get_record_service()
identity = get_identity_for_user(user)
- service = RecordService()
service.require_permission(identity, "delete")
for fi in (f for f in FileInstance.query.all() if not f.objects):
diff --git a/invenio_utilities_tuw/cli/record.py b/invenio_utilities_tuw/cli/records.py
similarity index 66%
rename from invenio_utilities_tuw/cli/record.py
rename to invenio_utilities_tuw/cli/records.py
index f4aebdd..840e54e 100644
--- a/invenio_utilities_tuw/cli/record.py
+++ b/invenio_utilities_tuw/cli/records.py
@@ -5,17 +5,12 @@ import json
import click
from flask.cli import with_appcontext
from invenio_files_rest.models import ObjectVersion
-from invenio_rdm_records.records.models import RecordMetadata
-from invenio_rdm_records.services.services import (
- BibliographicRecordService as RecordService,
-)
-from invenio_rdm_records.services.services import (
- BibliographicRecordFilesService as RecordFileService,
-)
+from ..utils import get_record_file_service, get_record_service
from .utils import (
convert_to_recid,
get_identity_for_user,
+ get_object_uuid,
patch_metadata,
)
@@ -42,26 +37,37 @@ option_pid_value = click.option(
"pid",
metavar="PID_VALUE",
required=True,
- help="persistent identifier of the record draft to operate on",
+ help="persistent identifier of the record to operate on",
+)
+option_pid_values = click.option(
+ "--pid",
+ "-p",
+ "pids",
+ metavar="PID_VALUE",
+ required=False,
+ multiple=True,
+ help="persistent identifier of the record to operate on (can be specified multiple times)",
)
@click.group()
-def record():
+def records():
"""Utility commands for creation and publication of drafts."""
pass
-@record.command("list")
+@records.command("list")
@option_as_user
@with_appcontext
def list_records(user):
"""List all records accessible to the given user."""
identity = get_identity_for_user(user)
- service = RecordService()
+ service = get_record_service()
+ rec_model_cls = service.record_cls.model_cls
+
recids = [
rec.json["id"]
- for rec in RecordMetadata.query.all()
+ for rec in rec_model_cls.query
if rec is not None and rec.json is not None
]
@@ -75,7 +81,7 @@ def list_records(user):
raise
-@record.command("update")
+@records.command("update")
@click.argument("metadata_file", type=click.File("r"))
@option_pid_value
@option_pid_type
@@ -91,7 +97,7 @@ def update_record(metadata_file, pid, pid_type, user, patch):
"""Update the specified draft's metadata."""
pid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = RecordService()
+ service = get_record_service()
metadata = json.load(metadata_file)
if patch:
@@ -102,7 +108,7 @@ def update_record(metadata_file, pid, pid_type, user, patch):
click.secho(pid, fg="green")
-@record.command("delete")
+@records.command("delete")
@click.confirmation_option(prompt="are you sure you want to delete this record?")
@option_pid_value
@option_pid_type
@@ -112,13 +118,13 @@ def delete_record(pid, pid_type, user):
"""Delete the specified record."""
identity = get_identity_for_user(user)
recid = convert_to_recid(pid, pid_type)
- service = RecordService()
+ service = get_record_service()
service.delete(id_=recid, identity=identity)
click.secho(recid, fg="red")
-@record.command("files")
+@records.command("files")
@option_pid_value
@option_pid_type
@option_as_user
@@ -127,9 +133,38 @@ def list_files(pid, pid_type, user):
"""Show a list of files deposited with the record."""
recid = convert_to_recid(pid, pid_type)
identity = get_identity_for_user(user)
- service = RecordFileService()
+ service = get_record_file_service()
file_results = service.list_files(id_=recid, identity=identity)
for f in file_results.entries:
ov = ObjectVersion.get(f["bucket_id"], f["key"], f["version_id"])
fi = ov.file
click.secho("{}\t{}\t{}".format(ov.key, fi.uri, fi.checksum), fg="green")
+
+
+@records.command("reindex")
+@option_pid_values
+@option_pid_type
+@option_as_user
+@with_appcontext
+def reindex_records(pids, pid_type, user):
+ """Reindex all available (or just the specified) records."""
+ service = get_record_service()
+
+ # basically, this is just a check whether the user exists,
+ # since there's no permission for re-indexing
+ get_identity_for_user(user)
+
+ if pids:
+ records = [
+ service.record_cls.get_record(get_object_uuid(pid, pid_type))
+ for pid in pids
+ ]
+ else:
+ records = [
+ service.record_cls.get_record(meta.id)
+ for meta in service.record_cls.model_cls.query
+ if meta is not None and meta.json is not None
+ ]
+
+ for record in records:
+ service.indexer.index(record)
diff --git a/invenio_utilities_tuw/cli/utils.py b/invenio_utilities_tuw/cli/utils.py
index 2798346..095e5ad 100644
--- a/invenio_utilities_tuw/cli/utils.py
+++ b/invenio_utilities_tuw/cli/utils.py
@@ -7,9 +7,8 @@ from invenio_access import any_user
from invenio_access.utils import get_identity
from invenio_accounts import current_accounts
from invenio_pidstore.models import PersistentIdentifier
-from invenio_rdm_records.services.services import (
- BibliographicRecordService as RecordService,
-)
+
+from ..utils import get_record_service
def create_record_from_metadata(metadata_file_path, identity):
@@ -21,7 +20,7 @@ def create_record_from_metadata(metadata_file_path, identity):
if metadata is None:
raise Exception("not a valid json file: %s" % metadata_file_path)
- service = RecordService()
+ service = get_record_service()
draft = service.create(identity=identity, data=metadata)
return draft
@@ -60,13 +59,25 @@ def get_identity_for_user(user):
return identity
+def get_object_uuid(pid_value, pid_type):
+ """Fetch the UUID of the referenced object."""
+ uuid = (
+ PersistentIdentifier.query.filter_by(pid_value=pid_value, pid_type=pid_type)
+ .first()
+ .object_uuid
+ )
+
+ return uuid
+
+
def convert_to_recid(pid_value, pid_type):
"""Fetch the recid of the referenced object."""
if pid_type != "recid":
- pid_value = (
- PersistentIdentifier.query.filter_by(pid_value=pid_value, pid_type=pid_type)
- .first()
- .pid_value
+ object_uuid = get_object_uuid(pid_value=pid_value, pid_type=pid_type)
+ query = PersistentIdentifier.query.filter_by(
+ object_uuid=object_uuid,
+ pid_type="recid",
)
+ pid_value = query.first().pid_value
return pid_value
diff --git a/invenio_utilities_tuw/config.py b/invenio_utilities_tuw/config.py
index 4cb4ab8..8f80f09 100644
--- a/invenio_utilities_tuw/config.py
+++ b/invenio_utilities_tuw/config.py
@@ -8,11 +8,23 @@
"""Some utilities for InvenioRDM."""
-# TODO: This is an example file. Remove it if your package does not use any
-# extra configuration variables.
+from invenio_rdm_records.proxies import current_rdm_records
-UTILITIES_TUW_DEFAULT_VALUE = "foobar"
-"""Default value for the application."""
UTILITIES_TUW_BASE_TEMPLATE = "invenio_utilities_tuw/base.html"
"""Default base template for the demo page."""
+
+UTILITIES_TUW_RECORD_SERVICE_FACTORY = (
+ lambda: current_rdm_records.records_service
+)
+"""Factory function for creating a RecordService."""
+
+UTILITIES_TUW_RECORD_FILES_SERVICE_FACTORY = (
+ lambda: current_rdm_records.record_files_service
+)
+"""Factory function for creating a RecordFileService."""
+
+UTILITIES_TUW_DRAFT_FILES_SERVICE_FACTORY = (
+ lambda: current_rdm_records.draft_files_service
+)
+"""Factory function for creating a DraftFileService."""
diff --git a/invenio_utilities_tuw/utils.py b/invenio_utilities_tuw/utils.py
new file mode 100644
index 0000000..dcf5389
--- /dev/null
+++ b/invenio_utilities_tuw/utils.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 TU Wien.
+#
+# Invenio-Utilities-TUW is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Utility functions for Invenio-Utilities-TUW."""
+
+
+from flask import current_app
+from invenio_rdm_records.proxies import current_rdm_records
+from werkzeug.utils import import_string
+
+
+def get_or_import(value, default=None):
+ """Try an import if value is an endpoint string, or return value itself."""
+ if isinstance(value, str):
+ return import_string(value)
+ elif value:
+ return value
+
+ return default
+
+
+def get_record_service():
+ """Get the configured RecordService."""
+ factory = current_app.config.get(
+ "UTILITIES_TUW_RECORD_SERVICE_FACTORY",
+ lambda: current_rdm_records.records_service,
+ )
+ return factory()
+
+
+def get_record_file_service():
+ """Get the configured RecordFileService."""
+ factory = current_app.config.get(
+ "UTILITIES_TUW_RECORD_FILES_SERVICE_FACTORY",
+ lambda: current_rdm_records.record_files_service,
+ )
+ return factory()
+
+
+def get_draft_file_service():
+ """Get the configured DraftFilesService."""
+ factory = current_app.config.get(
+ "UTILITIES_TUW_DRAFT_FILES_SERVICE_FACTORY",
+ lambda: current_rdm_records.draft_files_service,
+ )
+ return factory()
--
GitLab