From 2c10e3a20a8994ce16f88da582b85531030970c1 Mon Sep 17 00:00:00 2001
From: Lukas Mahler <e11908553@student.tuwien.ac.at>
Date: Wed, 27 Sep 2023 08:51:08 +0200
Subject: [PATCH] remove repositum
---
definitions.py | 2 +-
fairnb/entity/dbrepo_entity.py | 3 -
fairnb/entity/repositum_entity.py | 26 ---------
.../5_ml_model-checkpoint.ipynb | 4 +-
.../standalone-checkpoint.ipynb | 10 ++--
notebooks/main.ipynb | 57 ++-----------------
notebooks/standalone.ipynb | 10 ++--
test/integration/test_download_entities.py | 4 +-
test/test.ipynb | 2 +-
test/test_dbrepo.py | 4 +-
10 files changed, 23 insertions(+), 99 deletions(-)
delete mode 100644 fairnb/entity/repositum_entity.py
diff --git a/definitions.py b/definitions.py
index ce81d90..2d2ec96 100644
--- a/definitions.py
+++ b/definitions.py
@@ -3,4 +3,4 @@ import pathlib
BASE_PATH = pathlib.Path(__file__).parent.resolve()
BASE_DIR = BASE_PATH.as_posix()
CONFIG_PATH = pathlib.Path(__file__).parent.absolute() / "config"
-RESOURCE_PATH = BASE_PATH / "resource"
\ No newline at end of file
+RESOURCE_PATH = BASE_PATH / "resource"
diff --git a/fairnb/entity/dbrepo_entity.py b/fairnb/entity/dbrepo_entity.py
index b516db2..ae59e9e 100644
--- a/fairnb/entity/dbrepo_entity.py
+++ b/fairnb/entity/dbrepo_entity.py
@@ -1,5 +1,4 @@
import pandas as pd
-import numpy as np
from dataclasses import dataclass, field
from pathlib import Path
from fairnb.entity.entity import Entity
@@ -60,8 +59,6 @@ class DbRepoEntity(Entity):
# add id column to df:
df["entity_id"] = -1 # temporary, needed for easy table schema retrieval
- # TODO: reference Artefact id via foreign keys in data tables and dependency table
-
# create table if not exists
# update uri etc
# create table if it not already exists
diff --git a/fairnb/entity/repositum_entity.py b/fairnb/entity/repositum_entity.py
deleted file mode 100644
index e324f75..0000000
--- a/fairnb/entity/repositum_entity.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from pathlib import Path
-
-from fairnb.entity.entity import Entity
-from fairnb.entity.entity_provenance import EntityProvenance
-
-
-class RepositumEntity(Entity):
- @classmethod
- def existing(cls, *args, **kwargs):
- pass
-
- # TODO: add implementation of RepositumArtefact
-
- @classmethod
- def new(cls):
- pass
-
- def upload(self, executed_file: Path, dependencies: list[Entity] = None):
- pass
-
- def upload_data(self):
- pass
-
- def download(self) -> EntityProvenance:
- pass
-
diff --git a/notebooks/.ipynb_checkpoints/5_ml_model-checkpoint.ipynb b/notebooks/.ipynb_checkpoints/5_ml_model-checkpoint.ipynb
index 4153d28..533a691 100644
--- a/notebooks/.ipynb_checkpoints/5_ml_model-checkpoint.ipynb
+++ b/notebooks/.ipynb_checkpoints/5_ml_model-checkpoint.ipynb
@@ -27,9 +27,9 @@
"from sklearn.svm import SVC\n",
"\n",
"from definitions import BASE_PATH\n",
- "from dbrepo_ismir.util import Util\n",
+ "from fairnb.util import Util\n",
"\n",
- "from dbrepo_ismir.dbrepo import DBRepoConnector"
+ "from fairnb.dbrepo import DBRepoConnector"
]
},
{
diff --git a/notebooks/.ipynb_checkpoints/standalone-checkpoint.ipynb b/notebooks/.ipynb_checkpoints/standalone-checkpoint.ipynb
index 06486e7..8052009 100644
--- a/notebooks/.ipynb_checkpoints/standalone-checkpoint.ipynb
+++ b/notebooks/.ipynb_checkpoints/standalone-checkpoint.ipynb
@@ -32,11 +32,11 @@
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.svm import SVC\n",
"\n",
- "from dbrepo_ismir.entity.dbrepo_entity import DbRepoEntity\n",
- "from dbrepo_ismir.entity.invenio_entity import InvenioEntity\n",
- "from dbrepo_ismir.executor import Executor\n",
- "from dbrepo_ismir.nb_config import NbConfig\n",
- "from dbrepo_ismir.util import Util\n",
+ "from fairnb.entity.dbrepo_entity import DbRepoEntity\n",
+ "from fairnb.entity.invenio_entity import InvenioEntity\n",
+ "from fairnb.executor import Executor\n",
+ "from fairnb.nb_config import NbConfig\n",
+ "from fairnb.util import Util\n",
"from definitions import BASE_PATH, RESOURCE_PATH\n",
"import tarfile\n",
"import zipfile\n",
diff --git a/notebooks/main.ipynb b/notebooks/main.ipynb
index 5aaff9e..4423d8f 100644
--- a/notebooks/main.ipynb
+++ b/notebooks/main.ipynb
@@ -43,12 +43,11 @@
"import yaml\n",
"\n",
"from definitions import CONFIG_PATH, BASE_PATH, RESOURCE_PATH\n",
- "from dbrepo_ismir.entity.dbrepo_entity import DbRepoEntity\n",
- "from dbrepo_ismir.entity.invenio_entity import InvenioEntity\n",
- "from dbrepo_ismir.entity.repositum_entity import RepositumEntity\n",
- "from dbrepo_ismir.nb_config import NbConfig\n",
- "from dbrepo_ismir.executor import Executor\n",
- "from dbrepo_ismir.util import Util"
+ "from fairnb.entity.dbrepo_entity import DbRepoEntity\n",
+ "from fairnb.entity.invenio_entity import InvenioEntity\n",
+ "from fairnb.nb_config import NbConfig\n",
+ "from fairnb.executor import Executor\n",
+ "from fairnb.util import Util"
]
},
{
@@ -380,52 +379,6 @@
"# run ml\n",
"executor.execute(nb_config_ml, only_local=False)"
]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2023-09-01T11:35:31.931725838Z",
- "start_time": "2023-09-01T11:35:31.571858239Z"
- }
- },
- "outputs": [
- {
- "ename": "AttributeError",
- "evalue": "'NoneType' object has no attribute 'type'",
- "output_type": "error",
- "traceback": [
- "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
- "\u001B[0;31mAttributeError\u001B[0m Traceback (most recent call last)",
- "Cell \u001B[0;32mIn[10], line 18\u001B[0m\n\u001B[1;32m 2\u001B[0m nb_config_report \u001B[38;5;241m=\u001B[39m NbConfig(\n\u001B[1;32m 3\u001B[0m nb_location\u001B[38;5;241m=\u001B[39mNOTEBOOK_PATH \u001B[38;5;241m/\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m6_report.ipynb\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 4\u001B[0m entities\u001B[38;5;241m=\u001B[39m[\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 14\u001B[0m ]\n\u001B[1;32m 15\u001B[0m )\n\u001B[1;32m 17\u001B[0m \u001B[38;5;66;03m# upload report\u001B[39;00m\n\u001B[0;32m---> 18\u001B[0m \u001B[43mreproducer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mexecute\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnb_config_report\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43monly_local\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n",
- "File \u001B[0;32m~/Programming/uni/bachelorarbeit/dbrepo-ismir/dbrepo_ismir/executor.py:38\u001B[0m, in \u001B[0;36mExecutor.execute\u001B[0;34m(self, nb_config, require_download, only_local, **kwargs)\u001B[0m\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_preprocess(nb_config, require_download, only_local)\n\u001B[1;32m 30\u001B[0m \u001B[38;5;66;03m# papermill executes the notebook and inserts INPUT_PATHS and OUTPUT_PATHS into the notebook by either\u001B[39;00m\n\u001B[1;32m 31\u001B[0m \u001B[38;5;66;03m# replacing the cell tagged with 'parameters' or by inserting a new cell at the top of the notebook\u001B[39;00m\n\u001B[1;32m 32\u001B[0m \u001B[38;5;66;03m# with that tag.\u001B[39;00m\n\u001B[1;32m 33\u001B[0m papermill\u001B[38;5;241m.\u001B[39mexecute_notebook(\n\u001B[1;32m 34\u001B[0m nb_config\u001B[38;5;241m.\u001B[39mnb_location\u001B[38;5;241m.\u001B[39mresolve(),\n\u001B[1;32m 35\u001B[0m nb_config\u001B[38;5;241m.\u001B[39mnb_location\u001B[38;5;241m.\u001B[39mresolve(),\n\u001B[1;32m 36\u001B[0m parameters\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mdict\u001B[39m(\n\u001B[1;32m 37\u001B[0m INPUT_PATHS\u001B[38;5;241m=\u001B[39m{entity\u001B[38;5;241m.\u001B[39mtype: entity\u001B[38;5;241m.\u001B[39mlocation\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__str__\u001B[39m() \u001B[38;5;28;01mfor\u001B[39;00m entity \u001B[38;5;129;01min\u001B[39;00m nb_config\u001B[38;5;241m.\u001B[39mdependencies},\n\u001B[0;32m---> 38\u001B[0m OUTPUT_PATHS\u001B[38;5;241m=\u001B[39m{entity\u001B[38;5;241m.\u001B[39mtype: entity\u001B[38;5;241m.\u001B[39mlocation\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__str__\u001B[39m() \u001B[38;5;28;01mfor\u001B[39;00m entity \u001B[38;5;129;01min\u001B[39;00m nb_config\u001B[38;5;241m.\u001B[39mentities}\n\u001B[1;32m 39\u001B[0m )\n\u001B[1;32m 40\u001B[0m )\n\u001B[1;32m 42\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_post_process(nb_config, only_local)\n",
- "File \u001B[0;32m~/Programming/uni/bachelorarbeit/dbrepo-ismir/dbrepo_ismir/executor.py:38\u001B[0m, in \u001B[0;36m<dictcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_preprocess(nb_config, require_download, only_local)\n\u001B[1;32m 30\u001B[0m \u001B[38;5;66;03m# papermill executes the notebook and inserts INPUT_PATHS and OUTPUT_PATHS into the notebook by either\u001B[39;00m\n\u001B[1;32m 31\u001B[0m \u001B[38;5;66;03m# replacing the cell tagged with 'parameters' or by inserting a new cell at the top of the notebook\u001B[39;00m\n\u001B[1;32m 32\u001B[0m \u001B[38;5;66;03m# with that tag.\u001B[39;00m\n\u001B[1;32m 33\u001B[0m papermill\u001B[38;5;241m.\u001B[39mexecute_notebook(\n\u001B[1;32m 34\u001B[0m nb_config\u001B[38;5;241m.\u001B[39mnb_location\u001B[38;5;241m.\u001B[39mresolve(),\n\u001B[1;32m 35\u001B[0m nb_config\u001B[38;5;241m.\u001B[39mnb_location\u001B[38;5;241m.\u001B[39mresolve(),\n\u001B[1;32m 36\u001B[0m parameters\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mdict\u001B[39m(\n\u001B[1;32m 37\u001B[0m INPUT_PATHS\u001B[38;5;241m=\u001B[39m{entity\u001B[38;5;241m.\u001B[39mtype: entity\u001B[38;5;241m.\u001B[39mlocation\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__str__\u001B[39m() \u001B[38;5;28;01mfor\u001B[39;00m entity \u001B[38;5;129;01min\u001B[39;00m nb_config\u001B[38;5;241m.\u001B[39mdependencies},\n\u001B[0;32m---> 38\u001B[0m OUTPUT_PATHS\u001B[38;5;241m=\u001B[39m{\u001B[43mentity\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtype\u001B[49m: entity\u001B[38;5;241m.\u001B[39mlocation\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__str__\u001B[39m() \u001B[38;5;28;01mfor\u001B[39;00m entity \u001B[38;5;129;01min\u001B[39;00m nb_config\u001B[38;5;241m.\u001B[39mentities}\n\u001B[1;32m 39\u001B[0m )\n\u001B[1;32m 40\u001B[0m )\n\u001B[1;32m 42\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_post_process(nb_config, only_local)\n",
- "\u001B[0;31mAttributeError\u001B[0m: 'NoneType' object has no attribute 'type'"
- ]
- }
- ],
- "source": [
- "# # -------------- REPORT -----------------------\n",
- "# nb_config_report = NbConfig(\n",
- "# nb_location=NOTEBOOK_PATH / \"6_report.ipynb\",\n",
- "# entities=[\n",
- "# report_entity := RepositumEntity.new(\n",
- "# # id=None,\n",
- "# # location=LOCAL_PATH / \"6_report\" / \"report.pdf\",\n",
- "# # dbrepo_connector=connector\n",
- "# )\n",
- "# ],\n",
- "# dependencies=[\n",
- "# ml_model_entity.copy_to_location(LOCAL_PATH / \"6_report\" / \"ml_model.pickle\"),\n",
- "# test_result_entity.copy_to_location(LOCAL_PATH / \"6_report\" / \"test_result.csv\")\n",
- "# ]\n",
- "# )\n",
- "# \n",
- "# # upload report\n",
- "# executor.execute(nb_config_report, only_local=True)"
- ]
}
],
"metadata": {
diff --git a/notebooks/standalone.ipynb b/notebooks/standalone.ipynb
index 06486e7..8052009 100644
--- a/notebooks/standalone.ipynb
+++ b/notebooks/standalone.ipynb
@@ -32,11 +32,11 @@
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.svm import SVC\n",
"\n",
- "from dbrepo_ismir.entity.dbrepo_entity import DbRepoEntity\n",
- "from dbrepo_ismir.entity.invenio_entity import InvenioEntity\n",
- "from dbrepo_ismir.executor import Executor\n",
- "from dbrepo_ismir.nb_config import NbConfig\n",
- "from dbrepo_ismir.util import Util\n",
+ "from fairnb.entity.dbrepo_entity import DbRepoEntity\n",
+ "from fairnb.entity.invenio_entity import InvenioEntity\n",
+ "from fairnb.executor import Executor\n",
+ "from fairnb.nb_config import NbConfig\n",
+ "from fairnb.util import Util\n",
"from definitions import BASE_PATH, RESOURCE_PATH\n",
"import tarfile\n",
"import zipfile\n",
diff --git a/test/integration/test_download_entities.py b/test/integration/test_download_entities.py
index d99f512..55d1c4b 100644
--- a/test/integration/test_download_entities.py
+++ b/test/integration/test_download_entities.py
@@ -1,8 +1,8 @@
import yaml
from pytest import fixture
-from dbrepo_ismir.api.dbrepo import DBRepoConnector
-from dbrepo_ismir.entity.dbrepo_entity import DbRepoEntity
+from fairnb.api.dbrepo import DBRepoConnector
+from fairnb.entity.dbrepo_entity import DbRepoEntity
from definitions import BASE_PATH
diff --git a/test/test.ipynb b/test/test.ipynb
index 85ea41d..018a660 100644
--- a/test/test.ipynb
+++ b/test/test.ipynb
@@ -56,7 +56,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from dbrepo_ismir.dbrepo import DBRepoConnector"
+ "from fairnb.dbrepo import DBRepoConnector"
]
},
{
diff --git a/test/test_dbrepo.py b/test/test_dbrepo.py
index 74b1333..91c66d7 100644
--- a/test/test_dbrepo.py
+++ b/test/test_dbrepo.py
@@ -2,7 +2,7 @@ import unittest
import pandas as pd
from unittest import mock
import yaml
-from dbrepo_ismir.api.dbrepo import DBRepoConnector
+from fairnb.api.dbrepo import DBRepoConnector
from definitions import BASE_PATH
@@ -59,7 +59,7 @@ class DBRepoConnectorTester(unittest.TestCase):
assert self.connector_wrong_pw.__token is None
@mock.patch(
- 'dbrepo_ismir.dbrepo.DBRepoConnector.download_table',
+ 'fairnb.dbrepo.DBRepoConnector.download_table',
mocked_download_table
)
def test_download_table_as_df(self):
--
GitLab