From 69f47db917f3975fbae0ffc4051880de12609ad6 Mon Sep 17 00:00:00 2001
From: Maximilian Moser <maximilian.moser@tuwien.ac.at>
Date: Thu, 22 Feb 2024 17:46:59 +0100
Subject: [PATCH] Add resultman, a TUI tool for managing formatscaper results
* the initial implementation is limited to only viewing the list of
formats and listing the files (grouped by records) that use each
format
---
formatscaper/Pipfile | 1 +
formatscaper/Pipfile.lock | 25 +++-
formatscaper/resultman.py | 248 ++++++++++++++++++++++++++++++++++++++
3 files changed, 270 insertions(+), 4 deletions(-)
create mode 100755 formatscaper/resultman.py
diff --git a/formatscaper/Pipfile b/formatscaper/Pipfile
index af383c7..bda311c 100644
--- a/formatscaper/Pipfile
+++ b/formatscaper/Pipfile
@@ -6,6 +6,7 @@ name = "pypi"
[packages]
pyyaml = "*"
progressbar2 = "*"
+urwid = "*"
[dev-packages]
diff --git a/formatscaper/Pipfile.lock b/formatscaper/Pipfile.lock
index a8cc6ae..24de050 100644
--- a/formatscaper/Pipfile.lock
+++ b/formatscaper/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "f17599e0484b4226bebde0b6b444e0c48dcac21c3258f6354ef585800fdd2ab4"
+ "sha256": "a5e084bb30ee94af6dce803be60cad5d735281bfaebfb46d70a85380b0e4c130"
},
"pipfile-spec": 6,
"requires": {
@@ -27,11 +27,11 @@
},
"python-utils": {
"hashes": [
- "sha256:ec3a672465efb6c673845a43afcfafaa23d2594c24324a40ec18a0c59478dc0b",
- "sha256:efdf31c8154667d7dc0317547c8e6d3b506c5d4b6e360e0c89662306262fc0ab"
+ "sha256:ad0ccdbd6f856d015cace07f74828b9840b5c4072d9e868a7f6a14fd195555a8",
+ "sha256:c5d161e4ca58ce3f8c540f035e018850b261a41e7cb98f6ccf8e1deb7174a1f1"
],
"markers": "python_version >= '3.9'",
- "version": "==3.8.1"
+ "version": "==3.8.2"
},
"pyyaml": {
"hashes": [
@@ -64,6 +64,7 @@
"sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
"sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
"sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
+ "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
"sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
"sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
"sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
@@ -97,6 +98,22 @@
],
"markers": "python_version >= '3.8'",
"version": "==4.9.0"
+ },
+ "urwid": {
+ "hashes": [
+ "sha256:bc302170fdbdda0aded2787ba66006af939dcff967606e9840a6f2af149adf12",
+ "sha256:dea5b2e8f992946774fe864894d55c1045175d96c1266904755909dcd52d3cb5"
+ ],
+ "index": "pypi",
+ "markers": "python_version >= '3.8'",
+ "version": "==2.6.4"
+ },
+ "wcwidth": {
+ "hashes": [
+ "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859",
+ "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"
+ ],
+ "version": "==0.2.13"
}
},
"develop": {}
diff --git a/formatscaper/resultman.py b/formatscaper/resultman.py
new file mode 100755
index 0000000..c816e62
--- /dev/null
+++ b/formatscaper/resultman.py
@@ -0,0 +1,248 @@
+#!/bin/env python3
+
+import argparse
+import enum
+
+import urwid as uw
+from urwid.command_map import Command
+
+from core.utils import Format, load_formats, load_results
+
+
+# helper functions
+def fallback_key_handler(key: str) -> None:
+ """Handle keys that haven't been handled by any widgets."""
+ if key == "Q":
+ raise uw.ExitMainLoop()
+
+ elif key in ["q", "h", "esc"]:
+ # ideally these keys would be handled by the right side when it's focused
+ # as this action only makes sense then
+ columns.set_focus(left)
+
+
+class FormatFilter(enum.Enum):
+ """Filter settings for file formats."""
+
+ ALL = "all"
+ SAFE = "safe"
+ ENDANGERED = "endangered"
+
+
+class SText(uw.Text):
+ """A selectable Text widget."""
+
+ _selectable = True
+
+ def keypress(self, size, key):
+ """Don't do anything on keypress."""
+ return key
+
+
+class SimpleButton(uw.Button):
+ """Button widget with simpler decoration."""
+
+ button_left = uw.Text("*")
+ button_right = uw.Text(" ")
+
+
+# parsing CLI arguments
+parser = argparse.ArgumentParser(
+ description="TUI tool for managing file format information"
+)
+parser.add_argument(
+ "--formats",
+ "-f",
+ default="formats.yaml",
+ help="formats file",
+)
+parser.add_argument(
+ "--results",
+ "-r",
+ default="results.pickle",
+ help="results file",
+)
+parser.add_argument(
+ "--filter",
+ default=FormatFilter.ALL.value,
+ choices=[f.value for f in FormatFilter],
+ help="filter for formats based on their risk",
+)
+parser.add_argument(
+ "--invenio-domain",
+ default="researchdata.tuwien.ac.at",
+ help="domain name of the repository for building links",
+)
+args = parser.parse_args()
+current_filter = FormatFilter(args.filter)
+
+
+# loading formats & results
+formats, all_results = [], []
+try:
+ formats = list(load_formats(args.formats).values())
+except Exception as e:
+ print(e)
+
+try:
+ all_results = load_results(args.results) or []
+ all_results_per_record = {
+ rec: [res for res in all_results if res.record == rec]
+ for rec in {res.record for res in all_results}
+ }
+except Exception as e:
+ all_results, all_results_per_record = [], {}
+ print(e)
+
+
+# color palette & settings
+palette = [
+ ("border", "light gray,bold", "dark gray"),
+ ("bold", "bold", ""),
+ ("darkbg", "", "black"),
+ ("reversed", "bold,standout", ""),
+ ("endangered", "white", "dark red"),
+]
+
+uw.command_map["j"] = Command.DOWN
+uw.command_map["k"] = Command.UP
+uw.command_map["ctrl d"] = Command.PAGE_DOWN
+uw.command_map["ctrl u"] = Command.PAGE_UP
+
+PRONOM_BASE_URL = "https://www.nationalarchives.gov.uk/PRONOM/%(puid)s"
+REPOSITORY_BASE_URL = f"https://{args.invenio_domain}/records/%(recid)s"
+
+
+# event handlers
+def handle_select_format(format: Format, button: uw.Button):
+ """Set up the format details side (right) based on the selected format."""
+ relevant_results = {
+ rec: sorted(ress, key=lambda r: r.filename)
+ for rec, ress in {
+ rec: [r for r in ress if r.format == format]
+ for rec, ress in all_results_per_record.items()
+ }.items()
+ if ress
+ }
+
+ content = []
+ i, num_files = 0, 0
+ for rec, results in relevant_results.items():
+ i += 1
+ repository_url = REPOSITORY_BASE_URL % {"recid": rec}
+ content.append(uw.Filler(uw.Text([f" {i}) ", ("bold", repository_url)])))
+
+ for res in results:
+ content.append(
+ uw.AttrMap(
+ SText([" * ", res.filename], wrap="any"),
+ None,
+ focus_map="reversed",
+ )
+ )
+ content.append(uw.Divider())
+ num_files += len(results)
+
+ bottom = uw.SolidFill(".")
+ if relevant_results:
+ _files = "file" if num_files == 1 else "files"
+ _records = "record" if len(relevant_results) == 1 else "records"
+ bottom = uw.ScrollBar(
+ uw.ListBox(
+ uw.SimpleFocusListWalker(
+ [
+ uw.Divider(),
+ uw.AttrMap(
+ uw.Text(
+ f" {num_files} {_files} in "
+ f"{len(relevant_results)} {_records}:"
+ ),
+ None,
+ focus_map="reversed",
+ ),
+ uw.Divider(),
+ *content,
+ ]
+ )
+ )
+ )
+
+ pronom_url = PRONOM_BASE_URL % {"puid": format.puid}
+ format_header = uw.Pile(
+ [
+ uw.Filler(uw.Divider()),
+ uw.Filler(uw.Text([" Name: ", format.name])),
+ uw.Filler(uw.Text([" MIME: ", format.mime or "-"])),
+ uw.Filler(uw.Text([" PRONOM: ", pronom_url])),
+ uw.Filler(uw.Divider()),
+ ]
+ )
+
+ format_details = uw.Pile(
+ [
+ ("pack", format_header),
+ ("pack", uw.AttrMap(uw.Filler(uw.Divider()), "border")),
+ bottom,
+ ]
+ )
+
+ right.contents.pop()
+ right.contents.append((format_details, ("weight", 1)))
+
+ # if the details side has actual content to display, focus it
+ # columns: contains the format list (left) and the format details (right)
+ # right: has the format info header, divider, and files list
+ if relevant_results:
+ columns.set_focus(right)
+ right.set_focus(format_details)
+
+
+# defining the basic layout
+def create_format_buttons(filter: FormatFilter):
+ format_buttons = []
+ for format in sorted((f for f in formats if f.name), key=lambda f: f.name):
+ if filter == FormatFilter.ENDANGERED and not format.endangered:
+ continue
+ elif filter == FormatFilter.SAFE and format.endangered:
+ continue
+
+ attribute = "endangered" if format.endangered else "darkbg"
+ button = SimpleButton(format.name)
+ uw.connect_signal(button, "click", handle_select_format, user_args=[format])
+ format_buttons.append(uw.AttrMap(button, attribute, focus_map="reversed"))
+
+ return format_buttons
+
+
+formats = uw.ScrollBar(
+ uw.ListBox(uw.SimpleFocusListWalker(create_format_buttons(current_filter)))
+)
+formats._command_map["l"] = "activate"
+formats_label = uw.Filler(uw.AttrMap(uw.Text("FORMATS", align="center"), "border"))
+left = uw.Pile([("pack", formats_label), formats])
+
+help_text = """
+Formatscaper results manager help
+
+Up/Down/j/k/^d/^u: navigate up and down
+Enter/Space/l: select current format
+Esc/h/q: go back to formats list
+Q: quit
+"""
+details = uw.AttrMap(uw.Filler(uw.Text(help_text, align="center")), "darkbg")
+details_label = uw.Filler(uw.AttrMap(uw.Text("DETAILS", align="center"), "border"))
+right = uw.Pile([("pack", details_label), details])
+
+div = uw.AttrMap(uw.SolidFill(" "), "border")
+columns = uw.Columns([("weight", 1, left), (1, div), ("weight", 2, right)])
+
+filter_info = uw.Text(f"Showing: {current_filter.value} formats")
+status_line = uw.Filler(uw.AttrMap(filter_info, "border"))
+top = uw.Pile([uw.AttrMap(columns, "darkbg"), ("pack", status_line)])
+
+loop = uw.MainLoop(top, palette, unhandled_input=fallback_key_handler)
+
+try:
+ loop.run()
+except KeyboardInterrupt:
+ loop.stop()
--
GitLab