From 765de505bbe8b7c4741cdf78264985273dea37f0 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 20 Jan 2024 16:34:10 +0100 Subject: [PATCH] refactor: Remove AnnotatedDocument class The AnnotatedDocument class was, essentially, a simple tuple of a document and a list of annotations. While not bad in a vacuum, it is unwieldy and passing this around instead of a document, annotations, or both where necessary is more restrictive and frankly unnecessary. This commit removes the data class and any instances of its use. Instead, we now pass the individual components around to anything that needs them. This also frees us up to pass only annotations around for example. We also do not iterate through the selected papis documents to work on in each exporter anymore (since we only pass a single document), but in the main function itself. This leads to less duplication and makes the overall run function the overall single source of iteration through selected documents. Everything else only knows about a single document - the one it is operating on - which seems much neater. For now, it does not change much, but should make later work on extra exporters or extractors easier. --- papis_extract/__init__.py | 38 ++++++----- papis_extract/annotation.py | 12 ---- papis_extract/exporter.py | 27 ++++---- papis_extract/extractor.py | 42 ++++++------ papis_extract/formatter.py | 128 +++++++++++++++++++----------------- tests/test_formatting.py | 33 ++++------ 6 files changed, 142 insertions(+), 138 deletions(-) diff --git a/papis_extract/__init__.py b/papis_extract/__init__.py index 31d720d..8d7bf8a 100644 --- a/papis_extract/__init__.py +++ b/papis_extract/__init__.py @@ -7,7 +7,8 @@ import papis.notes import papis.strings from papis.document import Document -from papis_extract import extractor, exporter +from papis_extract import exporter, extractor +from papis_extract.annotation import Annotation from papis_extract.formatter import Formatter, formatters logger = papis.logging.get_logger(__name__) @@ -44,7 +45,7 @@ papis.config.register_default_settings(DEFAULT_OPTIONS) "--template", "-t", type=click.Choice( - ["markdown", "markdown-setext", "markdown-atx", "count", "csv"], + list(formatters.keys()), case_sensitive=False, ), help="Choose an output template to format annotations with.", @@ -85,27 +86,34 @@ def main( logger.warning(papis.strings.no_documents_retrieved_message) return - formatter = formatters[template] + formatter = formatters.get(template) run(documents, edit=manual, write=write, git=git, formatter=formatter, force=force) def run( documents: list[Document], - formatter: Formatter, + formatter: Formatter | None, edit: bool = False, write: bool = False, git: bool = False, force: bool = False, ) -> None: - annotated_docs = extractor.start(documents) - if write: - exporter.to_notes( - formatter=formatter, - annotated_docs=annotated_docs, - edit=edit, - git=git, - force=force, - ) - else: - exporter.to_stdout(formatter=formatter, annotated_docs=annotated_docs) + for doc in documents: + annotations: list[Annotation] = extractor.start(doc) + + if write: + exporter.to_notes( + formatter=formatter or formatters["markdown-atx"], + document=doc, + annotations=annotations, + edit=edit, + git=git, + force=force, + ) + else: + exporter.to_stdout( + formatter=formatter or formatters["markdown"], + document=doc, + annotations=annotations, + ) diff --git a/papis_extract/annotation.py b/papis_extract/annotation.py index 138b755..74342da 100644 --- a/papis_extract/annotation.py +++ b/papis_extract/annotation.py @@ -81,15 +81,3 @@ class Annotation: difference between full black and full white, as a float. """ return 1 - (abs(math.dist([*color_one], [*color_two])) / 3) - - -@dataclass -class AnnotatedDocument: - """Contains all annotations belonging to a single papis document. - - Combines a document with a list of annotations which belong to it.""" - - document: Document - annotations: list[Annotation] - - # TODO could implement a from_doc() static method to generate annotation list? diff --git a/papis_extract/exporter.py b/papis_extract/exporter.py index 80adb8c..120224e 100644 --- a/papis_extract/exporter.py +++ b/papis_extract/exporter.py @@ -6,27 +6,33 @@ import papis.api import papis.git import papis.config import Levenshtein -from papis_extract.annotation import AnnotatedDocument +from papis_extract.annotation import Annotation from papis_extract.formatter import Formatter logger = papis.logging.get_logger(__name__) -def to_stdout(formatter: Formatter, annotated_docs: list[AnnotatedDocument]) -> None: +def to_stdout( + formatter: Formatter, + document: papis.document.Document, + annotations: list[Annotation], +) -> None: """Pretty print annotations to stdout. Gives a nice human-readable representations of the annotations in somewhat of a list form. Not intended for machine-readability. """ - output: str = formatter(annotated_docs) - print(output.rstrip("\n")) + output: str = formatter(document, annotations) + if output: + print(output.rstrip("\n")) def to_notes( formatter: Formatter, - annotated_docs: list[AnnotatedDocument], + document: papis.document.Document, + annotations: list[Annotation], edit: bool, git: bool, force: bool, @@ -37,13 +43,12 @@ def to_notes( belonging to papis documents. Creates new notes for documents missing a note field or appends to existing. """ - for entry in annotated_docs: - formatted_annotations = formatter([entry]).split("\n") - if formatted_annotations: - _add_annots_to_note(entry.document, formatted_annotations, force=force) + formatted_annotations = formatter(document, annotations).split("\n") + if formatted_annotations: + _add_annots_to_note(document, formatted_annotations, force=force) - if edit: - papis.commands.edit.edit_notes(entry.document, git=git) + if edit: + papis.commands.edit.edit_notes(document, git=git) def _add_annots_to_note( diff --git a/papis_extract/extractor.py b/papis_extract/extractor.py index 3ae1650..180af25 100644 --- a/papis_extract/extractor.py +++ b/papis_extract/extractor.py @@ -10,41 +10,39 @@ import papis.config import papis.document from papis.document import Document -from papis_extract.annotation import Annotation, AnnotatedDocument +from papis_extract.annotation import Annotation logger = papis.logging.get_logger(__name__) def start( - documents: list[Document], -) -> list[AnnotatedDocument]: + document: Document, +) -> list[Annotation]: """Extract all annotations from passed documents. Returns all annotations contained in the papis documents passed in. """ - output: list[AnnotatedDocument] = [] - for doc in documents: - annotations: list[Annotation] = [] - found_pdf: bool = False - for file in doc.get_files(): - fname = Path(file) - if not _is_file_processable(fname): - break - found_pdf = True + annotations: list[Annotation] = [] + found_pdf: bool = False + for file in document.get_files(): + fname = Path(file) + if not _is_file_processable(fname): + break + found_pdf = True - try: - annotations.extend(extract(fname)) - except fitz.FileDataError as e: - print(f"File structure errors for {file}.\n{e}") + try: + annotations.extend(extract(fname)) + except fitz.FileDataError as e: + print(f"File structure errors for {file}.\n{e}") - if not found_pdf: - # have to remove curlys or papis logger gets upset - desc = re.sub("[{}]", "", papis.document.describe(doc)) - logger.warning("Did not find suitable PDF file for document: " f"{desc}") - output.append(AnnotatedDocument(doc, annotations)) - return output + if not found_pdf: + # have to remove curlys or papis logger gets upset + desc = re.sub("[{}]", "", papis.document.describe(document)) + logger.warning("Did not find suitable PDF file for document: " f"{desc}") + + return annotations def extract(filename: Path) -> list[Annotation]: diff --git a/papis_extract/formatter.py b/papis_extract/formatter.py index 75ed556..917e04c 100644 --- a/papis_extract/formatter.py +++ b/papis_extract/formatter.py @@ -1,86 +1,96 @@ from collections.abc import Callable -from papis_extract.annotation import AnnotatedDocument +from papis.document import Document -Formatter = Callable[[list[AnnotatedDocument]], str] +from papis_extract.annotation import Annotation + +Formatter = Callable[[Document, list[Annotation]], str] def format_markdown( - docs: list[AnnotatedDocument] = [], atx_headings: bool = False + document: Document = Document(), + annotations: list[Annotation] = [], + headings: str = "setext", # setext | atx | None ) -> str: + if not annotations: + return "" template = ( "{{#tag}}#{{tag}}\n{{/tag}}" "{{#quote}}> {{quote}}{{/quote}}{{#page}} [p. {{page}}]{{/page}}" "{{#note}}\n NOTE: {{note}}{{/note}}" ) output = "" - for entry in docs: - if not entry.annotations: - continue - heading = f"{entry.document['title']} - {entry.document['author']}\n" - if atx_headings: - output += f"# {heading}\n" - else: - title_decoration = ( - f"{'=' * len(entry.document.get('title', ''))} " - f"{'-' * len(entry.document.get('author', ''))}" - ) - output += f"{title_decoration}\n" f"{heading}" f"{title_decoration}\n\n" - - for a in entry.annotations: - output += a.format(template) - output += "\n\n" - - output += "\n\n\n" - - return output.rstrip() - - -def format_markdown_atx(docs: list[AnnotatedDocument] = []) -> str: - return format_markdown(docs, atx_headings=True) - - -def format_markdown_setext(docs: list[AnnotatedDocument] = []) -> str: - return format_markdown(docs, atx_headings=False) - - -def format_count(docs: list[AnnotatedDocument] = []) -> str: - output = "" - for entry in docs: - if not entry.annotations: - continue - - count = 0 - for _ in entry.annotations: - count += 1 - - d = entry.document - output += ( - f"{d['author'] if 'author' in d else ''}" - f"{' - ' if 'author' in d else ''}" # only put separator if author - f"{entry.document['title'] if 'title' in d else ''}: " - f"{count}\n" + heading = f"{document.get('title', '')} - {document.get('author', '')}" + if headings == "atx": + output = f"# {heading}\n\n" + elif headings == "setext": + title_decoration = ( + f"{'=' * len(document.get('title', ''))} " + f"{'-' * len(document.get('author', ''))}" ) + output = f"{title_decoration}\n{heading}\n{title_decoration}\n\n" + else: + output = "" + + for a in annotations: + output += a.format(template) + output += "\n\n" + + output += "\n\n\n" return output.rstrip() -def format_csv(docs: list[AnnotatedDocument] = []) -> str: +def format_markdown_atx( + document: Document = Document(), + annotations: list[Annotation] = [], +) -> str: + return format_markdown(document, annotations, headings="atx") + + +def format_markdown_setext( + document: Document = Document(), + annotations: list[Annotation] = [], +) -> str: + return format_markdown(document, annotations, headings="setext") + + +def format_count( + document: Document = Document(), + annotations: list[Annotation] = [], +) -> str: + if not annotations: + return "" + + count = 0 + for _ in annotations: + count += 1 + + return ( + f"{document.get('author', '')}" + f"{' - ' if 'author' in document else ''}" # only put separator if author + f"{document.get('title', '')}: " + f"{count}\n" + ).rstrip() + + +def format_csv( + document: Document = Document(), + annotations: list[Annotation] = [], +) -> str: header: str = "type,tag,page,quote,note,author,title,ref,file" template: str = ( '{{type}},{{tag}},{{page}},"{{quote}}","{{note}}",' '"{{doc.author}}","{{doc.title}}","{{doc.ref}}","{{file}}"' ) output = f"{header}\n" - for entry in docs: - if not entry.annotations: - continue + if not annotations: + return "" - d = entry.document - for a in entry.annotations: - output += a.format(template, doc=d) - output += "\n" + for a in annotations: + output += a.format(template, doc=document) + output += "\n" return output.rstrip() @@ -89,6 +99,6 @@ formatters: dict[str, Formatter] = { "count": format_count, "csv": format_csv, "markdown": format_markdown, - "markdown_atx": format_markdown_atx, - "markdown_setext": format_markdown_setext, + "markdown-atx": format_markdown_atx, + "markdown-setext": format_markdown_setext, } diff --git a/tests/test_formatting.py b/tests/test_formatting.py index d9a0d0f..244f4f8 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -1,5 +1,5 @@ from papis.document import Document -from papis_extract.annotation import AnnotatedDocument, Annotation +from papis_extract.annotation import Annotation from papis_extract.formatter import ( format_count, @@ -9,13 +9,11 @@ from papis_extract.formatter import ( format_markdown_setext, ) -an_doc: AnnotatedDocument = AnnotatedDocument( - Document(data={"author": "document-author", "title": "document-title"}), - [ - Annotation("myfile.pdf", text="my lovely text"), - Annotation("myfile.pdf", text="my second text", content="with note"), - ], -) +document = Document(data={"author": "document-author", "title": "document-title"}) +annotations = [ + Annotation("myfile.pdf", text="my lovely text"), + Annotation("myfile.pdf", text="my second text", content="with note"), +] md_default_output = """============== --------------- document-title - document-author ============== --------------- @@ -28,12 +26,12 @@ document-title - document-author def test_markdown_default(): fmt = format_markdown - assert fmt([an_doc]) == md_default_output + assert fmt(document, annotations) == md_default_output def test_markdown_atx(): fmt = format_markdown_atx - assert fmt([an_doc]) == ( + assert fmt(document, annotations) == ( """# document-title - document-author > my lovely text @@ -45,17 +43,17 @@ def test_markdown_atx(): def test_markdown_setext(): fmt = format_markdown_setext - assert fmt([an_doc]) == md_default_output + assert fmt(document, annotations) == md_default_output def test_count_default(): fmt = format_count - assert fmt([an_doc]) == ("""document-author - document-title: 2""") + assert fmt(document, annotations) == ("""document-author - document-title: 2""") def test_csv_default(): fmt = format_csv - assert fmt([an_doc]) == ( + assert fmt(document, annotations) == ( "type,tag,page,quote,note,author,title,ref,file\n" 'Highlight,,0,"my lovely text","","document-author",' '"document-title","","myfile.pdf"\n' @@ -66,15 +64,12 @@ def test_csv_default(): # sadpath - no annotations contained for each format def test_markdown_no_annotations(): - d: AnnotatedDocument = AnnotatedDocument(Document(data={}), []) - assert format_markdown([d]) == "" + assert format_markdown(document, []) == "" def test_count_no_annotations(): - d: AnnotatedDocument = AnnotatedDocument(Document(data={}), []) - assert format_count([d]) == "" + assert format_count(document, []) == "" def test_csv_no_annotations(): - d: AnnotatedDocument = AnnotatedDocument(Document(data={}), []) - assert format_csv([d]) == "type,tag,page,quote,note,author,title,ref,file" + assert format_csv(document, []) == ""