refactor: Remove AnnotatedDocument class
The AnnotatedDocument class was, essentially, a simple tuple of a document and a list of annotations. While not bad in a vacuum, it is unwieldy and passing this around instead of a document, annotations, or both where necessary is more restrictive and frankly unnecessary. This commit removes the data class and any instances of its use. Instead, we now pass the individual components around to anything that needs them. This also frees us up to pass only annotations around for example. We also do not iterate through the selected papis documents to work on in each exporter anymore (since we only pass a single document), but in the main function itself. This leads to less duplication and makes the overall run function the overall single source of iteration through selected documents. Everything else only knows about a single document - the one it is operating on - which seems much neater. For now, it does not change much, but should make later work on extra exporters or extractors easier.
This commit is contained in:
parent
cd5f787220
commit
765de505bb
GPG Key ID:
EDBF2ED917B2EF6A
|
|
@ -7,7 +7,8 @@ import papis.notes
|
|||
import papis.strings
|
||||
from papis.document import Document
|
||||
|
||||
from papis_extract import extractor, exporter
|
||||
from papis_extract import exporter, extractor
|
||||
from papis_extract.annotation import Annotation
|
||||
from papis_extract.formatter import Formatter, formatters
|
||||
|
||||
logger = papis.logging.get_logger(__name__)
|
||||
|
|
@ -44,7 +45,7 @@ papis.config.register_default_settings(DEFAULT_OPTIONS)
|
|||
"--template",
|
||||
"-t",
|
||||
type=click.Choice(
|
||||
["markdown", "markdown-setext", "markdown-atx", "count", "csv"],
|
||||
list(formatters.keys()),
|
||||
case_sensitive=False,
|
||||
),
|
||||
help="Choose an output template to format annotations with.",
|
||||
|
|
@ -85,27 +86,34 @@ def main(
|
|||
logger.warning(papis.strings.no_documents_retrieved_message)
|
||||
return
|
||||
|
||||
formatter = formatters[template]
|
||||
formatter = formatters.get(template)
|
||||
|
||||
run(documents, edit=manual, write=write, git=git, formatter=formatter, force=force)
|
||||
|
||||
|
||||
def run(
|
||||
documents: list[Document],
|
||||
formatter: Formatter,
|
||||
formatter: Formatter | None,
|
||||
edit: bool = False,
|
||||
write: bool = False,
|
||||
git: bool = False,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
annotated_docs = extractor.start(documents)
|
||||
if write:
|
||||
exporter.to_notes(
|
||||
formatter=formatter,
|
||||
annotated_docs=annotated_docs,
|
||||
edit=edit,
|
||||
git=git,
|
||||
force=force,
|
||||
)
|
||||
else:
|
||||
exporter.to_stdout(formatter=formatter, annotated_docs=annotated_docs)
|
||||
for doc in documents:
|
||||
annotations: list[Annotation] = extractor.start(doc)
|
||||
|
||||
if write:
|
||||
exporter.to_notes(
|
||||
formatter=formatter or formatters["markdown-atx"],
|
||||
document=doc,
|
||||
annotations=annotations,
|
||||
edit=edit,
|
||||
git=git,
|
||||
force=force,
|
||||
)
|
||||
else:
|
||||
exporter.to_stdout(
|
||||
formatter=formatter or formatters["markdown"],
|
||||
document=doc,
|
||||
annotations=annotations,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -81,15 +81,3 @@ class Annotation:
|
|||
difference between full black and full white, as a float.
|
||||
"""
|
||||
return 1 - (abs(math.dist([*color_one], [*color_two])) / 3)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnnotatedDocument:
|
||||
"""Contains all annotations belonging to a single papis document.
|
||||
|
||||
Combines a document with a list of annotations which belong to it."""
|
||||
|
||||
document: Document
|
||||
annotations: list[Annotation]
|
||||
|
||||
# TODO could implement a from_doc() static method to generate annotation list?
|
||||
|
|
|
|||
|
|
@ -6,27 +6,33 @@ import papis.api
|
|||
import papis.git
|
||||
import papis.config
|
||||
import Levenshtein
|
||||
from papis_extract.annotation import AnnotatedDocument
|
||||
from papis_extract.annotation import Annotation
|
||||
|
||||
from papis_extract.formatter import Formatter
|
||||
|
||||
logger = papis.logging.get_logger(__name__)
|
||||
|
||||
|
||||
def to_stdout(formatter: Formatter, annotated_docs: list[AnnotatedDocument]) -> None:
|
||||
def to_stdout(
|
||||
formatter: Formatter,
|
||||
document: papis.document.Document,
|
||||
annotations: list[Annotation],
|
||||
) -> None:
|
||||
"""Pretty print annotations to stdout.
|
||||
|
||||
Gives a nice human-readable representations of
|
||||
the annotations in somewhat of a list form.
|
||||
Not intended for machine-readability.
|
||||
"""
|
||||
output: str = formatter(annotated_docs)
|
||||
print(output.rstrip("\n"))
|
||||
output: str = formatter(document, annotations)
|
||||
if output:
|
||||
print(output.rstrip("\n"))
|
||||
|
||||
|
||||
def to_notes(
|
||||
formatter: Formatter,
|
||||
annotated_docs: list[AnnotatedDocument],
|
||||
document: papis.document.Document,
|
||||
annotations: list[Annotation],
|
||||
edit: bool,
|
||||
git: bool,
|
||||
force: bool,
|
||||
|
|
@ -37,13 +43,12 @@ def to_notes(
|
|||
belonging to papis documents. Creates new notes for
|
||||
documents missing a note field or appends to existing.
|
||||
"""
|
||||
for entry in annotated_docs:
|
||||
formatted_annotations = formatter([entry]).split("\n")
|
||||
if formatted_annotations:
|
||||
_add_annots_to_note(entry.document, formatted_annotations, force=force)
|
||||
formatted_annotations = formatter(document, annotations).split("\n")
|
||||
if formatted_annotations:
|
||||
_add_annots_to_note(document, formatted_annotations, force=force)
|
||||
|
||||
if edit:
|
||||
papis.commands.edit.edit_notes(entry.document, git=git)
|
||||
if edit:
|
||||
papis.commands.edit.edit_notes(document, git=git)
|
||||
|
||||
|
||||
def _add_annots_to_note(
|
||||
|
|
|
|||
|
|
@ -10,41 +10,39 @@ import papis.config
|
|||
import papis.document
|
||||
from papis.document import Document
|
||||
|
||||
from papis_extract.annotation import Annotation, AnnotatedDocument
|
||||
from papis_extract.annotation import Annotation
|
||||
|
||||
logger = papis.logging.get_logger(__name__)
|
||||
|
||||
|
||||
def start(
|
||||
documents: list[Document],
|
||||
) -> list[AnnotatedDocument]:
|
||||
document: Document,
|
||||
) -> list[Annotation]:
|
||||
"""Extract all annotations from passed documents.
|
||||
|
||||
Returns all annotations contained in the papis
|
||||
documents passed in.
|
||||
"""
|
||||
|
||||
output: list[AnnotatedDocument] = []
|
||||
for doc in documents:
|
||||
annotations: list[Annotation] = []
|
||||
found_pdf: bool = False
|
||||
for file in doc.get_files():
|
||||
fname = Path(file)
|
||||
if not _is_file_processable(fname):
|
||||
break
|
||||
found_pdf = True
|
||||
annotations: list[Annotation] = []
|
||||
found_pdf: bool = False
|
||||
for file in document.get_files():
|
||||
fname = Path(file)
|
||||
if not _is_file_processable(fname):
|
||||
break
|
||||
found_pdf = True
|
||||
|
||||
try:
|
||||
annotations.extend(extract(fname))
|
||||
except fitz.FileDataError as e:
|
||||
print(f"File structure errors for {file}.\n{e}")
|
||||
try:
|
||||
annotations.extend(extract(fname))
|
||||
except fitz.FileDataError as e:
|
||||
print(f"File structure errors for {file}.\n{e}")
|
||||
|
||||
if not found_pdf:
|
||||
# have to remove curlys or papis logger gets upset
|
||||
desc = re.sub("[{}]", "", papis.document.describe(doc))
|
||||
logger.warning("Did not find suitable PDF file for document: " f"{desc}")
|
||||
output.append(AnnotatedDocument(doc, annotations))
|
||||
return output
|
||||
if not found_pdf:
|
||||
# have to remove curlys or papis logger gets upset
|
||||
desc = re.sub("[{}]", "", papis.document.describe(document))
|
||||
logger.warning("Did not find suitable PDF file for document: " f"{desc}")
|
||||
|
||||
return annotations
|
||||
|
||||
|
||||
def extract(filename: Path) -> list[Annotation]:
|
||||
|
|
|
|||
|
|
@ -1,86 +1,96 @@
|
|||
from collections.abc import Callable
|
||||
|
||||
from papis_extract.annotation import AnnotatedDocument
|
||||
from papis.document import Document
|
||||
|
||||
Formatter = Callable[[list[AnnotatedDocument]], str]
|
||||
from papis_extract.annotation import Annotation
|
||||
|
||||
Formatter = Callable[[Document, list[Annotation]], str]
|
||||
|
||||
|
||||
def format_markdown(
|
||||
docs: list[AnnotatedDocument] = [], atx_headings: bool = False
|
||||
document: Document = Document(),
|
||||
annotations: list[Annotation] = [],
|
||||
headings: str = "setext", # setext | atx | None
|
||||
) -> str:
|
||||
if not annotations:
|
||||
return ""
|
||||
template = (
|
||||
"{{#tag}}#{{tag}}\n{{/tag}}"
|
||||
"{{#quote}}> {{quote}}{{/quote}}{{#page}} [p. {{page}}]{{/page}}"
|
||||
"{{#note}}\n NOTE: {{note}}{{/note}}"
|
||||
)
|
||||
output = ""
|
||||
for entry in docs:
|
||||
if not entry.annotations:
|
||||
continue
|
||||
|
||||
heading = f"{entry.document['title']} - {entry.document['author']}\n"
|
||||
if atx_headings:
|
||||
output += f"# {heading}\n"
|
||||
else:
|
||||
title_decoration = (
|
||||
f"{'=' * len(entry.document.get('title', ''))} "
|
||||
f"{'-' * len(entry.document.get('author', ''))}"
|
||||
)
|
||||
output += f"{title_decoration}\n" f"{heading}" f"{title_decoration}\n\n"
|
||||
|
||||
for a in entry.annotations:
|
||||
output += a.format(template)
|
||||
output += "\n\n"
|
||||
|
||||
output += "\n\n\n"
|
||||
|
||||
return output.rstrip()
|
||||
|
||||
|
||||
def format_markdown_atx(docs: list[AnnotatedDocument] = []) -> str:
|
||||
return format_markdown(docs, atx_headings=True)
|
||||
|
||||
|
||||
def format_markdown_setext(docs: list[AnnotatedDocument] = []) -> str:
|
||||
return format_markdown(docs, atx_headings=False)
|
||||
|
||||
|
||||
def format_count(docs: list[AnnotatedDocument] = []) -> str:
|
||||
output = ""
|
||||
for entry in docs:
|
||||
if not entry.annotations:
|
||||
continue
|
||||
|
||||
count = 0
|
||||
for _ in entry.annotations:
|
||||
count += 1
|
||||
|
||||
d = entry.document
|
||||
output += (
|
||||
f"{d['author'] if 'author' in d else ''}"
|
||||
f"{' - ' if 'author' in d else ''}" # only put separator if author
|
||||
f"{entry.document['title'] if 'title' in d else ''}: "
|
||||
f"{count}\n"
|
||||
heading = f"{document.get('title', '')} - {document.get('author', '')}"
|
||||
if headings == "atx":
|
||||
output = f"# {heading}\n\n"
|
||||
elif headings == "setext":
|
||||
title_decoration = (
|
||||
f"{'=' * len(document.get('title', ''))} "
|
||||
f"{'-' * len(document.get('author', ''))}"
|
||||
)
|
||||
output = f"{title_decoration}\n{heading}\n{title_decoration}\n\n"
|
||||
else:
|
||||
output = ""
|
||||
|
||||
for a in annotations:
|
||||
output += a.format(template)
|
||||
output += "\n\n"
|
||||
|
||||
output += "\n\n\n"
|
||||
|
||||
return output.rstrip()
|
||||
|
||||
|
||||
def format_csv(docs: list[AnnotatedDocument] = []) -> str:
|
||||
def format_markdown_atx(
|
||||
document: Document = Document(),
|
||||
annotations: list[Annotation] = [],
|
||||
) -> str:
|
||||
return format_markdown(document, annotations, headings="atx")
|
||||
|
||||
|
||||
def format_markdown_setext(
|
||||
document: Document = Document(),
|
||||
annotations: list[Annotation] = [],
|
||||
) -> str:
|
||||
return format_markdown(document, annotations, headings="setext")
|
||||
|
||||
|
||||
def format_count(
|
||||
document: Document = Document(),
|
||||
annotations: list[Annotation] = [],
|
||||
) -> str:
|
||||
if not annotations:
|
||||
return ""
|
||||
|
||||
count = 0
|
||||
for _ in annotations:
|
||||
count += 1
|
||||
|
||||
return (
|
||||
f"{document.get('author', '')}"
|
||||
f"{' - ' if 'author' in document else ''}" # only put separator if author
|
||||
f"{document.get('title', '')}: "
|
||||
f"{count}\n"
|
||||
).rstrip()
|
||||
|
||||
|
||||
def format_csv(
|
||||
document: Document = Document(),
|
||||
annotations: list[Annotation] = [],
|
||||
) -> str:
|
||||
header: str = "type,tag,page,quote,note,author,title,ref,file"
|
||||
template: str = (
|
||||
'{{type}},{{tag}},{{page}},"{{quote}}","{{note}}",'
|
||||
'"{{doc.author}}","{{doc.title}}","{{doc.ref}}","{{file}}"'
|
||||
)
|
||||
output = f"{header}\n"
|
||||
for entry in docs:
|
||||
if not entry.annotations:
|
||||
continue
|
||||
if not annotations:
|
||||
return ""
|
||||
|
||||
d = entry.document
|
||||
for a in entry.annotations:
|
||||
output += a.format(template, doc=d)
|
||||
output += "\n"
|
||||
for a in annotations:
|
||||
output += a.format(template, doc=document)
|
||||
output += "\n"
|
||||
|
||||
return output.rstrip()
|
||||
|
||||
|
|
@ -89,6 +99,6 @@ formatters: dict[str, Formatter] = {
|
|||
"count": format_count,
|
||||
"csv": format_csv,
|
||||
"markdown": format_markdown,
|
||||
"markdown_atx": format_markdown_atx,
|
||||
"markdown_setext": format_markdown_setext,
|
||||
"markdown-atx": format_markdown_atx,
|
||||
"markdown-setext": format_markdown_setext,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from papis.document import Document
|
||||
from papis_extract.annotation import AnnotatedDocument, Annotation
|
||||
from papis_extract.annotation import Annotation
|
||||
|
||||
from papis_extract.formatter import (
|
||||
format_count,
|
||||
|
|
@ -9,13 +9,11 @@ from papis_extract.formatter import (
|
|||
format_markdown_setext,
|
||||
)
|
||||
|
||||
an_doc: AnnotatedDocument = AnnotatedDocument(
|
||||
Document(data={"author": "document-author", "title": "document-title"}),
|
||||
[
|
||||
Annotation("myfile.pdf", text="my lovely text"),
|
||||
Annotation("myfile.pdf", text="my second text", content="with note"),
|
||||
],
|
||||
)
|
||||
document = Document(data={"author": "document-author", "title": "document-title"})
|
||||
annotations = [
|
||||
Annotation("myfile.pdf", text="my lovely text"),
|
||||
Annotation("myfile.pdf", text="my second text", content="with note"),
|
||||
]
|
||||
md_default_output = """============== ---------------
|
||||
document-title - document-author
|
||||
============== ---------------
|
||||
|
|
@ -28,12 +26,12 @@ document-title - document-author
|
|||
|
||||
def test_markdown_default():
|
||||
fmt = format_markdown
|
||||
assert fmt([an_doc]) == md_default_output
|
||||
assert fmt(document, annotations) == md_default_output
|
||||
|
||||
|
||||
def test_markdown_atx():
|
||||
fmt = format_markdown_atx
|
||||
assert fmt([an_doc]) == (
|
||||
assert fmt(document, annotations) == (
|
||||
"""# document-title - document-author
|
||||
|
||||
> my lovely text
|
||||
|
|
@ -45,17 +43,17 @@ def test_markdown_atx():
|
|||
|
||||
def test_markdown_setext():
|
||||
fmt = format_markdown_setext
|
||||
assert fmt([an_doc]) == md_default_output
|
||||
assert fmt(document, annotations) == md_default_output
|
||||
|
||||
|
||||
def test_count_default():
|
||||
fmt = format_count
|
||||
assert fmt([an_doc]) == ("""document-author - document-title: 2""")
|
||||
assert fmt(document, annotations) == ("""document-author - document-title: 2""")
|
||||
|
||||
|
||||
def test_csv_default():
|
||||
fmt = format_csv
|
||||
assert fmt([an_doc]) == (
|
||||
assert fmt(document, annotations) == (
|
||||
"type,tag,page,quote,note,author,title,ref,file\n"
|
||||
'Highlight,,0,"my lovely text","","document-author",'
|
||||
'"document-title","","myfile.pdf"\n'
|
||||
|
|
@ -66,15 +64,12 @@ def test_csv_default():
|
|||
|
||||
# sadpath - no annotations contained for each format
|
||||
def test_markdown_no_annotations():
|
||||
d: AnnotatedDocument = AnnotatedDocument(Document(data={}), [])
|
||||
assert format_markdown([d]) == ""
|
||||
assert format_markdown(document, []) == ""
|
||||
|
||||
|
||||
def test_count_no_annotations():
|
||||
d: AnnotatedDocument = AnnotatedDocument(Document(data={}), [])
|
||||
assert format_count([d]) == ""
|
||||
assert format_count(document, []) == ""
|
||||
|
||||
|
||||
def test_csv_no_annotations():
|
||||
d: AnnotatedDocument = AnnotatedDocument(Document(data={}), [])
|
||||
assert format_csv([d]) == "type,tag,page,quote,note,author,title,ref,file"
|
||||
assert format_csv(document, []) == ""
|
||||
|
|
|
|||
Loading…
Reference in New Issue