From b6a28d8e6aadb21b0e6e308c42e0128f3ded3b2f Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Wed, 22 Feb 2023 12:02:00 +0100 Subject: [PATCH 01/10] Add dict type to notes --- papis-marvin | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/papis-marvin b/papis-marvin index 58a3a00..b3f5019 100755 --- a/papis-marvin +++ b/papis-marvin @@ -6,7 +6,7 @@ # annotations as 'csv' format and then point the script to the # resulting file. import sys -from typing import List +from typing import Dict, List import papis.api import papis.commands.list import papis.database @@ -24,7 +24,7 @@ def main(fpath, db): write_to_files(notes) -def get_all_annotations(db, csv): +def get_all_annotations(db, csv) -> Dict: notes = {} note_file = "" for row in csv: @@ -54,6 +54,17 @@ def get_all_annotations(db, csv): return notes +def get_documents(db, author, title) -> List: + return db.query(f"author:({author}) title:({title})") + + +def get_notefile(document) -> str | None: + note_file = papis.commands.list.run(notes=True, documents=document) + if not note_file: + return + return str(note_file[0]) + + # TODO Implement manual note creation def manual_papis_entry(): if input(f"Create note file now? [y/N] ") == "y": @@ -68,17 +79,6 @@ def manual_papis_entry(): return -def get_documents(db, author, title) -> List: - return db.query(f"author:({author}) title:({title})") - - -def get_notefile(document) -> str | None: - note_file = papis.commands.list.run(notes=True, documents=document) - if not note_file: - return - return str(note_file[0]) - - # TODO implement custom formatting (akin to pubs-extract) def format_entry(row) -> str: text = f"> {row['HighlightText']}" @@ -104,7 +104,7 @@ def is_same_book(title): return False -def write_to_files(notes): +def write_to_files(notes: Dict): # write to notes for f, entries in notes.items(): if f: From 46ea1192040833b7d9dd6792b5c2b2904bb4cd60 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Wed, 22 Feb 2023 13:19:17 +0100 Subject: [PATCH 02/10] Add gitignore file --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..112f45c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +testfile.csv From 32d57c79d3a85669464114a1d43249860de81536 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Wed, 22 Feb 2023 13:19:34 +0100 Subject: [PATCH 03/10] Fix note file creation --- papis-marvin | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/papis-marvin b/papis-marvin index b3f5019..2485399 100755 --- a/papis-marvin +++ b/papis-marvin @@ -5,9 +5,14 @@ # the iOS application 'Marvin Reader'. In the app, export your # annotations as 'csv' format and then point the script to the # resulting file. +import os import sys from typing import Dict, List import papis.api +import papis.pick +import papis.format +import papis.commands.edit +import papis.config import papis.commands.list import papis.database import re @@ -38,12 +43,14 @@ def get_all_annotations(db, csv) -> Dict: f"No papis entry found for Marvin entry - {row['Author']}: {title_stripped}.\nPlease manually create." ) continue - note_file = get_notefile(documents) + # TODO warn user/ let him pick with picker if multiple docs found + doc = documents[0] + note_file = get_notefile(doc) if not note_file: print( f"Found reference entry but no note file for - {row['Author']}: {title_stripped}." ) - manual_papis_entry() + note_file = create_note_file(db, doc) text = format_entry(row) @@ -59,22 +66,29 @@ def get_documents(db, author, title) -> List: def get_notefile(document) -> str | None: - note_file = papis.commands.list.run(notes=True, documents=document) + note_file = papis.commands.list.run(notes=True, documents=[document]) if not note_file: return return str(note_file[0]) -# TODO Implement manual note creation -def manual_papis_entry(): +def create_note_file(db, document): if input(f"Create note file now? [y/N] ") == "y": - # i think instead we need to run the papis.edit cmd with note bool true and maybe editor set to none? - output = subprocess.run( - ["papis", "edit", "-n"], capture_output=True, shell=True + if not document.has("notes"): + notes_name = papis.config.getstring("notes-name") + document["notes"] = papis.format.format(notes_name, document) + document.save() + db.update(document) + + notes_path = os.path.join( + str(document.get_main_folder()), + document["notes"] ) - print(output) - print("NOT IMPLEMENTED: Please create note file manually.") - return + + if not os.path.exists(notes_path): + # TODO reimplement logger: logger.debug("Creating '%s'", notes_path) + papis.commands.edit.create_notes(document, notes_path) + return notes_path else: return From cb1cfe22abf02aee8fd604e6d1e6fed1e3056ac3 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 23 Feb 2023 10:33:54 +0100 Subject: [PATCH 04/10] Add automatic import for missing entries --- papis-marvin | 108 +++++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/papis-marvin b/papis-marvin index 2485399..408a1bb 100755 --- a/papis-marvin +++ b/papis-marvin @@ -7,16 +7,23 @@ # resulting file. import os import sys +import re +import logging +import subprocess from typing import Dict, List import papis.api import papis.pick import papis.format import papis.commands.edit -import papis.config import papis.commands.list +import papis.commands.add +import papis.config import papis.database -import re -import subprocess +import isbnlib +import papis.isbn + +logger = logging.getLogger("marvin") +logger.setLevel(logging.DEBUG) def main(fpath, db): @@ -33,24 +40,13 @@ def get_all_annotations(db, csv) -> Dict: notes = {} note_file = "" for row in csv: - title_stripped = strip_book_title(row["Title"]) - # switch to next book if not is_same_book(row["Title"]): - documents = get_documents(db, row["Author"], title_stripped) - if not documents: - print( - f"No papis entry found for Marvin entry - {row['Author']}: {title_stripped}.\nPlease manually create." - ) + title_stripped = strip_string(row["Title"]) + doc = get_document(db, row["Author"], title_stripped) + if not doc: continue - # TODO warn user/ let him pick with picker if multiple docs found - doc = documents[0] - note_file = get_notefile(doc) - if not note_file: - print( - f"Found reference entry but no note file for - {row['Author']}: {title_stripped}." - ) - note_file = create_note_file(db, doc) + note_file = get_notefile(db, doc) text = format_entry(row) @@ -61,36 +57,48 @@ def get_all_annotations(db, csv) -> Dict: return notes -def get_documents(db, author, title) -> List: - return db.query(f"author:({author}) title:({title})") +# TODO warn user/ let him pick with picker if multiple docs found +def get_document(db, author, title): + res = db.query(f"author:({author}) title:({title})") + if not res: + add_to_database(author, title) + res = db.query(f"author:({author}) title:({title})") + if not res: + logger.warning( + f"Nothing found for {author}: {title}.\nPlease create manually." + ) + return res[0] + return res[0] -def get_notefile(document) -> str | None: - note_file = papis.commands.list.run(notes=True, documents=[document]) - if not note_file: - return - return str(note_file[0]) - - -def create_note_file(db, document): - if input(f"Create note file now? [y/N] ") == "y": - if not document.has("notes"): - notes_name = papis.config.getstring("notes-name") - document["notes"] = papis.format.format(notes_name, document) - document.save() - db.update(document) - - notes_path = os.path.join( - str(document.get_main_folder()), - document["notes"] - ) - - if not os.path.exists(notes_path): - # TODO reimplement logger: logger.debug("Creating '%s'", notes_path) - papis.commands.edit.create_notes(document, notes_path) - return notes_path +def add_to_database(author, title, confirm=True, edit=False): + print(f"Searching - '{title} {author}'") + data = None + try: + data = papis.isbn.get_data(f"{title}") + except isbnlib.ISBNLibException as e: + print(f"ERROR: {e}") else: - return + print(f"Found: {data}") + if data: + papis_data = papis.isbn.data_to_papis(data[0]) + papis.commands.add.run([], data=papis_data, confirm=confirm, edit=edit) + + +def get_notefile(db, document) -> str | None: + if not document.has("notes"): + notes_name = papis.config.getstring("notes-name") + document["notes"] = papis.format.format(notes_name, document) + document.save() + db.update(document) + + notes_path = os.path.join(str(document.get_main_folder()), document["notes"]) + print(f"{document['title']}: {document['notes']}") + + if not os.path.exists(notes_path): + # TODO reimplement logger: logger.debug("Creating '%s'", notes_path) + papis.commands.edit.create_notes(document, notes_path) + return notes_path # TODO implement custom formatting (akin to pubs-extract) @@ -123,21 +131,21 @@ def write_to_files(notes: Dict): for f, entries in notes.items(): if f: with open(f, "a") as note: - print(f"Editing {f}...") + logger.info(f"Editing {f}...") num_added = 0 for entry in entries: with open(f) as noteread: if entry not in noteread.read(): note.write(f"{entry}\n\n") num_added += 1 - print(f"Added {num_added} entries to it.") + logger.info(f"Added {num_added} entries to it.") -title_strip_pattern = re.compile(r"([^\s\w]|_)+") +strip_pattern = re.compile(r"([^\s\w]|_)+") -def strip_book_title(title) -> str: - return title_strip_pattern.sub("", title) +def strip_string(title) -> str: + return strip_pattern.sub("", title) if __name__ == "__main__": From b161413f137b60617614132d119d963f7cd980e6 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 23 Feb 2023 19:29:31 +0100 Subject: [PATCH 05/10] Extract document querying function --- papis-marvin | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/papis-marvin b/papis-marvin index 408a1bb..8f2e6f9 100755 --- a/papis-marvin +++ b/papis-marvin @@ -57,18 +57,24 @@ def get_all_annotations(db, csv) -> Dict: return notes -# TODO warn user/ let him pick with picker if multiple docs found def get_document(db, author, title): - res = db.query(f"author:({author}) title:({title})") + res = query_document(db, author, title) if not res: add_to_database(author, title) - res = db.query(f"author:({author}) title:({title})") - if not res: - logger.warning( - f"Nothing found for {author}: {title}.\nPlease create manually." - ) - return res[0] - return res[0] + res = query_document(db, author, title) + if not res: + logger.warning(f"Nothing found for {author}: {title}.\nPlease create manually.") + return + return res + + +# TODO warn user/ let him pick with picker if multiple docs found +def query_document(db, author, title): + title = strip_string(title) + for query in [f"author:({author}) title:({title})"]: + res = db.query(query) + if len(res) >= 1: + return res[0] def add_to_database(author, title, confirm=True, edit=False): @@ -93,7 +99,6 @@ def get_notefile(db, document) -> str | None: db.update(document) notes_path = os.path.join(str(document.get_main_folder()), document["notes"]) - print(f"{document['title']}: {document['notes']}") if not os.path.exists(notes_path): # TODO reimplement logger: logger.debug("Creating '%s'", notes_path) From 94744fb55f7162b32f9b202c3c49b22e1985272d Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 23 Feb 2023 19:32:32 +0100 Subject: [PATCH 06/10] Only strip query title --- papis-marvin | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/papis-marvin b/papis-marvin index 8f2e6f9..77bdc33 100755 --- a/papis-marvin +++ b/papis-marvin @@ -42,8 +42,7 @@ def get_all_annotations(db, csv) -> Dict: for row in csv: # switch to next book if not is_same_book(row["Title"]): - title_stripped = strip_string(row["Title"]) - doc = get_document(db, row["Author"], title_stripped) + doc = get_document(db, row["Author"], row["Title"]) if not doc: continue note_file = get_notefile(db, doc) From a1502595384de4749c01a581901e36baa56364e0 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 23 Feb 2023 19:32:43 +0100 Subject: [PATCH 07/10] Move print statements to logger methods --- papis-marvin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/papis-marvin b/papis-marvin index 77bdc33..e9b248a 100755 --- a/papis-marvin +++ b/papis-marvin @@ -77,14 +77,14 @@ def query_document(db, author, title): def add_to_database(author, title, confirm=True, edit=False): - print(f"Searching - '{title} {author}'") + logger.info(f"Searching - '{title} {author}'") data = None try: data = papis.isbn.get_data(f"{title}") except isbnlib.ISBNLibException as e: - print(f"ERROR: {e}") + logger.error(e) else: - print(f"Found: {data}") + logger.info(f"Found: {data}") if data: papis_data = papis.isbn.data_to_papis(data[0]) papis.commands.add.run([], data=papis_data, confirm=confirm, edit=edit) From 82f2e7d7df357346e808aac937b911af5e98136e Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 23 Feb 2023 21:35:37 +0100 Subject: [PATCH 08/10] Trim everything after special character for titles --- papis-marvin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papis-marvin b/papis-marvin index e9b248a..1ddb00c 100755 --- a/papis-marvin +++ b/papis-marvin @@ -145,7 +145,7 @@ def write_to_files(notes: Dict): logger.info(f"Added {num_added} entries to it.") -strip_pattern = re.compile(r"([^\s\w]|_)+") +strip_pattern = re.compile(r"([^\s\w]|_)+\w*") def strip_string(title) -> str: From d4cbc2eb77a5a6e38b2560a206b3a146cc80ce9d Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Tue, 29 Aug 2023 22:41:25 +0200 Subject: [PATCH 09/10] Add default csv path variable --- papis-marvin | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/papis-marvin b/papis-marvin index 1ddb00c..4ce4ba1 100755 --- a/papis-marvin +++ b/papis-marvin @@ -9,14 +9,14 @@ import os import sys import re import logging -import subprocess -from typing import Dict, List +from typing import Dict import papis.api import papis.pick import papis.format import papis.commands.edit import papis.commands.list import papis.commands.add +import papis.notes import papis.config import papis.database import isbnlib @@ -25,6 +25,8 @@ import papis.isbn logger = logging.getLogger("marvin") logger.setLevel(logging.DEBUG) +DEFAULT_CSV_PATH = "/home/marty/Nextcloud/Personal/Backups/Journal.csv" + def main(fpath, db): with open(fpath) as f: @@ -50,7 +52,7 @@ def get_all_annotations(db, csv) -> Dict: text = format_entry(row) if note_file and text: - if not note_file in notes.keys(): + if note_file not in notes.keys(): notes[note_file] = [] notes[note_file].append(text) return notes @@ -60,7 +62,7 @@ def get_document(db, author, title): res = query_document(db, author, title) if not res: add_to_database(author, title) - res = query_document(db, author, title) + res = query_document(db, author, title) if not res: logger.warning(f"Nothing found for {author}: {title}.\nPlease create manually.") return @@ -71,6 +73,7 @@ def get_document(db, author, title): def query_document(db, author, title): title = strip_string(title) for query in [f"author:({author}) title:({title})"]: + print(f"query: {query}") res = db.query(query) if len(res) >= 1: return res[0] @@ -84,7 +87,7 @@ def add_to_database(author, title, confirm=True, edit=False): except isbnlib.ISBNLibException as e: logger.error(e) else: - logger.info(f"Found: {data}") + logger.warning(f"Found: {data}") if data: papis_data = papis.isbn.data_to_papis(data[0]) papis.commands.add.run([], data=papis_data, confirm=confirm, edit=edit) @@ -101,7 +104,7 @@ def get_notefile(db, document) -> str | None: if not os.path.exists(notes_path): # TODO reimplement logger: logger.debug("Creating '%s'", notes_path) - papis.commands.edit.create_notes(document, notes_path) + papis.notes.notes_path_ensured(document) return notes_path @@ -154,10 +157,6 @@ def strip_string(title) -> str: if __name__ == "__main__": # use argument passed to command as file or default file here - fpath = ( - sys.argv[1] - if len(sys.argv) > 1 - else "/home/marty/Nextcloud/Personal/Backups/Journal.csv" - ) + fpath = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_CSV_PATH main(fpath, papis.database.get()) From b00e7f17702d3fe416f80a4f8c8f85d92745ad81 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Tue, 29 Aug 2023 22:41:40 +0200 Subject: [PATCH 10/10] Integrate remote repo url into comment --- papis-marvin | 1 + 1 file changed, 1 insertion(+) diff --git a/papis-marvin b/papis-marvin index 4ce4ba1..479a4da 100755 --- a/papis-marvin +++ b/papis-marvin @@ -5,6 +5,7 @@ # the iOS application 'Marvin Reader'. In the app, export your # annotations as 'csv' format and then point the script to the # resulting file. +# https://git.martyoeh.me/Marty/papis-marvin import os import sys import re