refactor: Remove AnnotatedDocument class

The AnnotatedDocument class was, essentially, a simple tuple of a document
and a list of annotations. While not bad in a vacuum, it is unwieldy and
passing this around instead of a document, annotations, or both where
necessary is more restrictive and frankly unnecessary.

This commit removes the data class and any instances of its use. Instead,
we now pass the individual components around to anything that needs them.
This also frees us up to pass only annotations around for example.

We also do not iterate through the selected papis documents to work on
in each exporter anymore (since we only pass a single document), but
in the main function itself. This leads to less duplication and makes
the overall run function the overall single source of iteration through
selected documents. Everything else only knows about a single document -
the one it is operating on - which seems much neater.

For now, it does not change much, but should make later work on extra
exporters or extractors easier.
This commit is contained in:
Marty Oehme 2024-01-20 16:34:10 +01:00
parent cd5f787220
commit 765de505bb
Signed by: Marty
GPG key ID: EDBF2ED917B2EF6A
6 changed files with 142 additions and 138 deletions

View file

@ -7,7 +7,8 @@ import papis.notes
import papis.strings
from papis.document import Document
from papis_extract import extractor, exporter
from papis_extract import exporter, extractor
from papis_extract.annotation import Annotation
from papis_extract.formatter import Formatter, formatters
logger = papis.logging.get_logger(__name__)
@ -44,7 +45,7 @@ papis.config.register_default_settings(DEFAULT_OPTIONS)
"--template",
"-t",
type=click.Choice(
["markdown", "markdown-setext", "markdown-atx", "count", "csv"],
list(formatters.keys()),
case_sensitive=False,
),
help="Choose an output template to format annotations with.",
@ -85,27 +86,34 @@ def main(
logger.warning(papis.strings.no_documents_retrieved_message)
return
formatter = formatters[template]
formatter = formatters.get(template)
run(documents, edit=manual, write=write, git=git, formatter=formatter, force=force)
def run(
documents: list[Document],
formatter: Formatter,
formatter: Formatter | None,
edit: bool = False,
write: bool = False,
git: bool = False,
force: bool = False,
) -> None:
annotated_docs = extractor.start(documents)
if write:
exporter.to_notes(
formatter=formatter,
annotated_docs=annotated_docs,
edit=edit,
git=git,
force=force,
)
else:
exporter.to_stdout(formatter=formatter, annotated_docs=annotated_docs)
for doc in documents:
annotations: list[Annotation] = extractor.start(doc)
if write:
exporter.to_notes(
formatter=formatter or formatters["markdown-atx"],
document=doc,
annotations=annotations,
edit=edit,
git=git,
force=force,
)
else:
exporter.to_stdout(
formatter=formatter or formatters["markdown"],
document=doc,
annotations=annotations,
)

View file

@ -81,15 +81,3 @@ class Annotation:
difference between full black and full white, as a float.
"""
return 1 - (abs(math.dist([*color_one], [*color_two])) / 3)
@dataclass
class AnnotatedDocument:
"""Contains all annotations belonging to a single papis document.
Combines a document with a list of annotations which belong to it."""
document: Document
annotations: list[Annotation]
# TODO could implement a from_doc() static method to generate annotation list?

View file

@ -6,27 +6,33 @@ import papis.api
import papis.git
import papis.config
import Levenshtein
from papis_extract.annotation import AnnotatedDocument
from papis_extract.annotation import Annotation
from papis_extract.formatter import Formatter
logger = papis.logging.get_logger(__name__)
def to_stdout(formatter: Formatter, annotated_docs: list[AnnotatedDocument]) -> None:
def to_stdout(
formatter: Formatter,
document: papis.document.Document,
annotations: list[Annotation],
) -> None:
"""Pretty print annotations to stdout.
Gives a nice human-readable representations of
the annotations in somewhat of a list form.
Not intended for machine-readability.
"""
output: str = formatter(annotated_docs)
print(output.rstrip("\n"))
output: str = formatter(document, annotations)
if output:
print(output.rstrip("\n"))
def to_notes(
formatter: Formatter,
annotated_docs: list[AnnotatedDocument],
document: papis.document.Document,
annotations: list[Annotation],
edit: bool,
git: bool,
force: bool,
@ -37,13 +43,12 @@ def to_notes(
belonging to papis documents. Creates new notes for
documents missing a note field or appends to existing.
"""
for entry in annotated_docs:
formatted_annotations = formatter([entry]).split("\n")
if formatted_annotations:
_add_annots_to_note(entry.document, formatted_annotations, force=force)
formatted_annotations = formatter(document, annotations).split("\n")
if formatted_annotations:
_add_annots_to_note(document, formatted_annotations, force=force)
if edit:
papis.commands.edit.edit_notes(entry.document, git=git)
if edit:
papis.commands.edit.edit_notes(document, git=git)
def _add_annots_to_note(

View file

@ -10,41 +10,39 @@ import papis.config
import papis.document
from papis.document import Document
from papis_extract.annotation import Annotation, AnnotatedDocument
from papis_extract.annotation import Annotation
logger = papis.logging.get_logger(__name__)
def start(
documents: list[Document],
) -> list[AnnotatedDocument]:
document: Document,
) -> list[Annotation]:
"""Extract all annotations from passed documents.
Returns all annotations contained in the papis
documents passed in.
"""
output: list[AnnotatedDocument] = []
for doc in documents:
annotations: list[Annotation] = []
found_pdf: bool = False
for file in doc.get_files():
fname = Path(file)
if not _is_file_processable(fname):
break
found_pdf = True
annotations: list[Annotation] = []
found_pdf: bool = False
for file in document.get_files():
fname = Path(file)
if not _is_file_processable(fname):
break
found_pdf = True
try:
annotations.extend(extract(fname))
except fitz.FileDataError as e:
print(f"File structure errors for {file}.\n{e}")
try:
annotations.extend(extract(fname))
except fitz.FileDataError as e:
print(f"File structure errors for {file}.\n{e}")
if not found_pdf:
# have to remove curlys or papis logger gets upset
desc = re.sub("[{}]", "", papis.document.describe(doc))
logger.warning("Did not find suitable PDF file for document: " f"{desc}")
output.append(AnnotatedDocument(doc, annotations))
return output
if not found_pdf:
# have to remove curlys or papis logger gets upset
desc = re.sub("[{}]", "", papis.document.describe(document))
logger.warning("Did not find suitable PDF file for document: " f"{desc}")
return annotations
def extract(filename: Path) -> list[Annotation]:

View file

@ -1,86 +1,96 @@
from collections.abc import Callable
from papis_extract.annotation import AnnotatedDocument
from papis.document import Document
Formatter = Callable[[list[AnnotatedDocument]], str]
from papis_extract.annotation import Annotation
Formatter = Callable[[Document, list[Annotation]], str]
def format_markdown(
docs: list[AnnotatedDocument] = [], atx_headings: bool = False
document: Document = Document(),
annotations: list[Annotation] = [],
headings: str = "setext", # setext | atx | None
) -> str:
if not annotations:
return ""
template = (
"{{#tag}}#{{tag}}\n{{/tag}}"
"{{#quote}}> {{quote}}{{/quote}}{{#page}} [p. {{page}}]{{/page}}"
"{{#note}}\n NOTE: {{note}}{{/note}}"
)
output = ""
for entry in docs:
if not entry.annotations:
continue
heading = f"{entry.document['title']} - {entry.document['author']}\n"
if atx_headings:
output += f"# {heading}\n"
else:
title_decoration = (
f"{'=' * len(entry.document.get('title', ''))} "
f"{'-' * len(entry.document.get('author', ''))}"
)
output += f"{title_decoration}\n" f"{heading}" f"{title_decoration}\n\n"
for a in entry.annotations:
output += a.format(template)
output += "\n\n"
output += "\n\n\n"
return output.rstrip()
def format_markdown_atx(docs: list[AnnotatedDocument] = []) -> str:
return format_markdown(docs, atx_headings=True)
def format_markdown_setext(docs: list[AnnotatedDocument] = []) -> str:
return format_markdown(docs, atx_headings=False)
def format_count(docs: list[AnnotatedDocument] = []) -> str:
output = ""
for entry in docs:
if not entry.annotations:
continue
count = 0
for _ in entry.annotations:
count += 1
d = entry.document
output += (
f"{d['author'] if 'author' in d else ''}"
f"{' - ' if 'author' in d else ''}" # only put separator if author
f"{entry.document['title'] if 'title' in d else ''}: "
f"{count}\n"
heading = f"{document.get('title', '')} - {document.get('author', '')}"
if headings == "atx":
output = f"# {heading}\n\n"
elif headings == "setext":
title_decoration = (
f"{'=' * len(document.get('title', ''))} "
f"{'-' * len(document.get('author', ''))}"
)
output = f"{title_decoration}\n{heading}\n{title_decoration}\n\n"
else:
output = ""
for a in annotations:
output += a.format(template)
output += "\n\n"
output += "\n\n\n"
return output.rstrip()
def format_csv(docs: list[AnnotatedDocument] = []) -> str:
def format_markdown_atx(
document: Document = Document(),
annotations: list[Annotation] = [],
) -> str:
return format_markdown(document, annotations, headings="atx")
def format_markdown_setext(
document: Document = Document(),
annotations: list[Annotation] = [],
) -> str:
return format_markdown(document, annotations, headings="setext")
def format_count(
document: Document = Document(),
annotations: list[Annotation] = [],
) -> str:
if not annotations:
return ""
count = 0
for _ in annotations:
count += 1
return (
f"{document.get('author', '')}"
f"{' - ' if 'author' in document else ''}" # only put separator if author
f"{document.get('title', '')}: "
f"{count}\n"
).rstrip()
def format_csv(
document: Document = Document(),
annotations: list[Annotation] = [],
) -> str:
header: str = "type,tag,page,quote,note,author,title,ref,file"
template: str = (
'{{type}},{{tag}},{{page}},"{{quote}}","{{note}}",'
'"{{doc.author}}","{{doc.title}}","{{doc.ref}}","{{file}}"'
)
output = f"{header}\n"
for entry in docs:
if not entry.annotations:
continue
if not annotations:
return ""
d = entry.document
for a in entry.annotations:
output += a.format(template, doc=d)
output += "\n"
for a in annotations:
output += a.format(template, doc=document)
output += "\n"
return output.rstrip()
@ -89,6 +99,6 @@ formatters: dict[str, Formatter] = {
"count": format_count,
"csv": format_csv,
"markdown": format_markdown,
"markdown_atx": format_markdown_atx,
"markdown_setext": format_markdown_setext,
"markdown-atx": format_markdown_atx,
"markdown-setext": format_markdown_setext,
}

View file

@ -1,5 +1,5 @@
from papis.document import Document
from papis_extract.annotation import AnnotatedDocument, Annotation
from papis_extract.annotation import Annotation
from papis_extract.formatter import (
format_count,
@ -9,13 +9,11 @@ from papis_extract.formatter import (
format_markdown_setext,
)
an_doc: AnnotatedDocument = AnnotatedDocument(
Document(data={"author": "document-author", "title": "document-title"}),
[
Annotation("myfile.pdf", text="my lovely text"),
Annotation("myfile.pdf", text="my second text", content="with note"),
],
)
document = Document(data={"author": "document-author", "title": "document-title"})
annotations = [
Annotation("myfile.pdf", text="my lovely text"),
Annotation("myfile.pdf", text="my second text", content="with note"),
]
md_default_output = """============== ---------------
document-title - document-author
============== ---------------
@ -28,12 +26,12 @@ document-title - document-author
def test_markdown_default():
fmt = format_markdown
assert fmt([an_doc]) == md_default_output
assert fmt(document, annotations) == md_default_output
def test_markdown_atx():
fmt = format_markdown_atx
assert fmt([an_doc]) == (
assert fmt(document, annotations) == (
"""# document-title - document-author
> my lovely text
@ -45,17 +43,17 @@ def test_markdown_atx():
def test_markdown_setext():
fmt = format_markdown_setext
assert fmt([an_doc]) == md_default_output
assert fmt(document, annotations) == md_default_output
def test_count_default():
fmt = format_count
assert fmt([an_doc]) == ("""document-author - document-title: 2""")
assert fmt(document, annotations) == ("""document-author - document-title: 2""")
def test_csv_default():
fmt = format_csv
assert fmt([an_doc]) == (
assert fmt(document, annotations) == (
"type,tag,page,quote,note,author,title,ref,file\n"
'Highlight,,0,"my lovely text","","document-author",'
'"document-title","","myfile.pdf"\n'
@ -66,15 +64,12 @@ def test_csv_default():
# sadpath - no annotations contained for each format
def test_markdown_no_annotations():
d: AnnotatedDocument = AnnotatedDocument(Document(data={}), [])
assert format_markdown([d]) == ""
assert format_markdown(document, []) == ""
def test_count_no_annotations():
d: AnnotatedDocument = AnnotatedDocument(Document(data={}), [])
assert format_count([d]) == ""
assert format_count(document, []) == ""
def test_csv_no_annotations():
d: AnnotatedDocument = AnnotatedDocument(Document(data={}), [])
assert format_csv([d]) == "type,tag,page,quote,note,author,title,ref,file"
assert format_csv(document, []) == ""