From e4f1ee35910e0873638e19a8981ffb5e7bc9fdde Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 24 Dec 2022 18:01:55 +0100 Subject: [PATCH 1/9] Refactor annotations into dataclass --- extract/extract.py | 112 +++++++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/extract/extract.py b/extract/extract.py index f25bd73..18f3f0f 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -1,11 +1,14 @@ import os import re import argparse +from dataclasses import dataclass +from typing import Tuple import fitz import Levenshtein from pubs.plugins import PapersPlugin +from pubs.paper import Paper from pubs.events import DocAddEvent, NoteEvent from pubs import repo, pretty @@ -13,7 +16,45 @@ from pubs.utils import resolve_citekey_list from pubs.content import check_file, read_text_file, write_file from pubs.query import get_paper_filter -CONFIRMATION_PAPER_THRESHOLD=5 +CONFIRMATION_PAPER_THRESHOLD = 5 + + +@dataclass +class Annotation: + """A PDF annotation object""" + + paper: Paper + file: str + type: str = "Highlight" + text: str = "" + content: str = "" + page: int = 1 + colors: Tuple = (0.0, 0.0, 0.0) + + def formatted(self, formatting): + output = formatting + replacements = { + r"{quote}": self.text, + r"{note}": self.content, + r"{page}": str(self.page), + r"{newline}": "\n", + } + if self.text == "": + output = re.sub(r"{quote_begin}.*{quote_end}", "", output) + if self.content == "": + output = re.sub(r"{note_begin}.*{note_end}", "", output) + output = re.sub(r"{note_begin}", "", output) + output = re.sub(r"{note_end}", "", output) + output = re.sub(r"{quote_begin}", "", output) + output = re.sub(r"{quote_end}", "", output) + pattern = re.compile( + "|".join( + [re.escape(k) for k in sorted(replacements, key=len, reverse=True)] + ), + flags=re.DOTALL, + ) + return pattern.sub(lambda x: replacements[x.group(0)], output) + class ExtractPlugin(PapersPlugin): """Extract annotations from any pdf document. @@ -116,11 +157,12 @@ class ExtractPlugin(PapersPlugin): Returns all annotations belonging to the papers that are described by the citekeys passed in. """ - papers_annotated = [] + papers_annotated = {} for paper in papers: file = self._get_file(paper) try: - papers_annotated.append((paper, self._get_annotations(file))) + annotations = self._get_annotations(file, paper) + papers_annotated[paper.citekey] = annotations except fitz.FileDataError as e: self.ui.error(f"Document {file} is broken: {e}") return papers_annotated @@ -177,7 +219,7 @@ class ExtractPlugin(PapersPlugin): self.ui.warning(f"{paper.citekey} has no valid document.") return path - def _get_annotations(self, filename): + def _get_annotations(self, filename, paper): """Extract annotations from a file. Returns all readable annotations contained in the file @@ -190,34 +232,18 @@ class ExtractPlugin(PapersPlugin): for annot in page.annots(): quote, note = self._retrieve_annotation_content(page, annot) annotations.append( - self._format_annotation(quote, note, page.number or 0) + Annotation( + file=filename, + paper=paper, + text=quote, + content=note, + colors=annot.colors, + type=annot.type, + page=(page.number or 0) + 1, + ) ) return annotations - def _format_annotation(self, quote, note, pagenumber=0): - output = self.formatting - replacements = { - r"{quote}": quote, - r"{note}": note, - r"{page}": str(pagenumber), - r"{newline}": "\n", - } - if note == "": - output = re.sub(r"{note_begin}.*{note_end}", "", output) - if quote == "": - output = re.sub(r"{quote_begin}.*{quote_end}", "", output) - output = re.sub(r"{note_begin}", "", output) - output = re.sub(r"{note_end}", "", output) - output = re.sub(r"{quote_begin}", "", output) - output = re.sub(r"{quote_end}", "", output) - pattern = re.compile( - "|".join( - [re.escape(k) for k in sorted(replacements, key=len, reverse=True)] - ), - flags=re.DOTALL, - ) - return pattern.sub(lambda x: replacements[x.group(0)], output) - def _retrieve_annotation_content(self, page, annotation): """Gets the text content of an annotation. @@ -249,13 +275,11 @@ class ExtractPlugin(PapersPlugin): ready to be passed on through pipelines etc. """ output = "" - for contents in annotated_papers: - paper = contents[0] - annotations = contents[1] - if annotations: - output += f"------ {paper.citekey} ------\n" - for annot in annotations: - output += f"{annot}\n" + for citekey, annotations in annotated_papers.items(): + output += f"------ {citekey} ------\n" + for annotation in annotations: + # for annot in annotations: + output += f"{annotation.formatted(self.formatting)}\n" output += "\n" print(output) @@ -266,20 +290,18 @@ class ExtractPlugin(PapersPlugin): in the pubs notes directory. Creates new notes for citekeys missing a note or appends to existing. """ - for contents in annotated_papers: - paper = contents[0] - annotations = contents[1] + for citekey, annotations in annotated_papers.items(): if annotations: - notepath = self.broker.real_notepath(paper.citekey, note_extension) + notepath = self.broker.real_notepath(citekey, note_extension) if check_file(notepath, fail=False): self._append_to_note(notepath, annotations) else: self._write_new_note(notepath, annotations) - self.ui.info(f"Wrote annotations to {paper.citekey} note {notepath}.") + self.ui.info(f"Wrote annotations to {citekey} note {notepath}.") if edit is True: self.ui.edit_file(notepath, temporary=False) - NoteEvent(paper.citekey).send() + NoteEvent(citekey).send() def _write_new_note(self, notepath, annotations): """Create a new note containing the annotations. @@ -289,7 +311,7 @@ class ExtractPlugin(PapersPlugin): """ output = "# Annotations\n\n" for annotation in annotations: - output += f"{annotation}\n\n" + output += f"{annotation.formatted(self.formatting)}\n\n" write_file(notepath, output, "w") def _append_to_note(self, notepath, annotations): @@ -300,13 +322,13 @@ class ExtractPlugin(PapersPlugin): """ existing = read_text_file(notepath) # removed annotations already found in the note - existing_dropped = [x for x in annotations if x not in existing] + existing_dropped = [x for x in annotations if x.formatted(self.formatting) not in existing] if not existing_dropped: return output = "" for annotation in existing_dropped: - output += f"{annotation}\n\n" + output += f"{annotation.formatted(self.formatting)}\n\n" write_file(notepath, output, "a") From a62968beac492ef73239e9649de1a74c3065124c Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 24 Dec 2022 18:19:42 +0100 Subject: [PATCH 2/9] Add colorname extraction from annotations --- extract/extract.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/extract/extract.py b/extract/extract.py index 18f3f0f..8c95c63 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -1,6 +1,7 @@ import os import re import argparse +import math from dataclasses import dataclass from typing import Tuple @@ -18,6 +19,15 @@ from pubs.query import get_paper_filter CONFIRMATION_PAPER_THRESHOLD = 5 +COLORS = { + "red": (1, 0, 0), + "green": (0, 1, 0), + "blue": (0, 0, 1), + "yellow": (1, 1, 0), + "purple": (0.5, 0, 0.5), + "orange": (1, 0.65, 0), +} + @dataclass class Annotation: @@ -55,6 +65,17 @@ class Annotation: ) return pattern.sub(lambda x: replacements[x.group(0)], output) + @property + def colorname(self): + annot_colors = self.colors.get("stroke") or self.colors.get("fill") + nearest = None + smallest_dist = 2.0 + for name, values in COLORS.items(): + dist = math.dist([*values], [*annot_colors]) + if dist < smallest_dist: + smallest_dist = dist + nearest = name + return nearest class ExtractPlugin(PapersPlugin): """Extract annotations from any pdf document. @@ -86,6 +107,7 @@ class ExtractPlugin(PapersPlugin): "formatting", "{newline}{quote_begin}> {quote} {quote_end}[{page}]{note_begin}{newline}Note: {note}{note_end}", ) + self.color_mapping = settings.get("color_mapping", {}) def update_parser(self, subparsers, _): """Allow the usage of the pubs extract subcommand""" @@ -167,6 +189,9 @@ class ExtractPlugin(PapersPlugin): self.ui.error(f"Document {file} is broken: {e}") return papers_annotated + def mapped_tag(self, colorname): + return self.color_mapping.get(colorname) + def _gather_papers(self, conf, args): """Get all papers for citekeys. From 6bb7ed9c9ff1e5dca6a71f49ac6da5f17146c801 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 24 Dec 2022 18:21:30 +0100 Subject: [PATCH 3/9] Switch bare print to use ui message function --- extract/extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract/extract.py b/extract/extract.py index 8c95c63..d8a3c6f 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -306,7 +306,7 @@ class ExtractPlugin(PapersPlugin): # for annot in annotations: output += f"{annotation.formatted(self.formatting)}\n" output += "\n" - print(output) + self.ui.message(output) def _to_notes(self, annotated_papers, note_extension="txt", edit=False): """Write annotations into pubs notes. From 5bddf97e585292725ae833286ef580227d325fc3 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 24 Dec 2022 18:37:36 +0100 Subject: [PATCH 4/9] Add tags to formatted annotations --- extract/extract.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/extract/extract.py b/extract/extract.py index d8a3c6f..f470ed6 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -40,6 +40,7 @@ class Annotation: content: str = "" page: int = 1 colors: Tuple = (0.0, 0.0, 0.0) + tag: str = None def formatted(self, formatting): output = formatting @@ -48,6 +49,7 @@ class Annotation: r"{note}": self.content, r"{page}": str(self.page), r"{newline}": "\n", + r"{tag}": self.tag, } if self.text == "": output = re.sub(r"{quote_begin}.*{quote_end}", "", output) @@ -77,6 +79,7 @@ class Annotation: nearest = name return nearest + class ExtractPlugin(PapersPlugin): """Extract annotations from any pdf document. @@ -105,7 +108,7 @@ class ExtractPlugin(PapersPlugin): self.minimum_similarity = float(settings.get("minimum_similarity", 0.75)) self.formatting = settings.get( "formatting", - "{newline}{quote_begin}> {quote} {quote_end}[{page}]{note_begin}{newline}Note: {note}{note_end}", + "{newline}{quote_begin}> {quote} {quote_end}[{page}]{note_begin}{newline}Note: {note} {note_end} #{tag}", ) self.color_mapping = settings.get("color_mapping", {}) @@ -189,7 +192,7 @@ class ExtractPlugin(PapersPlugin): self.ui.error(f"Document {file} is broken: {e}") return papers_annotated - def mapped_tag(self, colorname): + def tag_from_colorname(self, colorname): return self.color_mapping.get(colorname) def _gather_papers(self, conf, args): @@ -256,17 +259,17 @@ class ExtractPlugin(PapersPlugin): for page in doc: for annot in page.annots(): quote, note = self._retrieve_annotation_content(page, annot) - annotations.append( - Annotation( - file=filename, - paper=paper, - text=quote, - content=note, - colors=annot.colors, - type=annot.type, - page=(page.number or 0) + 1, - ) + a = Annotation( + file=filename, + paper=paper, + text=quote, + content=note, + colors=annot.colors, + type=annot.type[1], + page=(page.number or 0) + 1, ) + a.tag = self.tag_from_colorname(a.colorname) + annotations.append(a) return annotations def _retrieve_annotation_content(self, page, annotation): From 2b476206a2fba9bb5749ba1e7f35b24f6c6310c2 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 24 Dec 2022 23:32:34 +0100 Subject: [PATCH 5/9] Change formatting configuration style for containers --- extract/extract.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/extract/extract.py b/extract/extract.py index f470ed6..9d4875b 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -2,8 +2,8 @@ import os import re import argparse import math -from dataclasses import dataclass -from typing import Tuple +from dataclasses import dataclass, field +from typing import Dict import fitz import Levenshtein @@ -39,8 +39,8 @@ class Annotation: text: str = "" content: str = "" page: int = 1 - colors: Tuple = (0.0, 0.0, 0.0) - tag: str = None + colors: Dict = field(default_factory=lambda: {"stroke": (0.0, 0.0, 0.0)}) + tag: str = "" def formatted(self, formatting): output = formatting @@ -51,25 +51,25 @@ class Annotation: r"{newline}": "\n", r"{tag}": self.tag, } - if self.text == "": - output = re.sub(r"{quote_begin}.*{quote_end}", "", output) - if self.content == "": - output = re.sub(r"{note_begin}.*{note_end}", "", output) - output = re.sub(r"{note_begin}", "", output) - output = re.sub(r"{note_end}", "", output) - output = re.sub(r"{quote_begin}", "", output) - output = re.sub(r"{quote_end}", "", output) pattern = re.compile( "|".join( [re.escape(k) for k in sorted(replacements, key=len, reverse=True)] ), flags=re.DOTALL, ) + patt_quote_container = re.compile(r"{%quote_container(.*?)%}") + patt_note_container = re.compile(r"{%note_container(.*?)%}") + patt_tag_container = re.compile(r"{%tag_container(.*?)%}") + output = patt_quote_container.sub(r"\1" if self.text else "", output) + output = patt_note_container.sub(r"\1" if self.content else "", output) + output = patt_tag_container.sub(r"\1" if self.tag else "", output) return pattern.sub(lambda x: replacements[x.group(0)], output) @property def colorname(self): - annot_colors = self.colors.get("stroke") or self.colors.get("fill") + annot_colors = ( + self.colors.get("stroke") or self.colors.get("fill") or (0.0, 0.0, 0.0) + ) nearest = None smallest_dist = 2.0 for name, values in COLORS.items(): @@ -108,7 +108,7 @@ class ExtractPlugin(PapersPlugin): self.minimum_similarity = float(settings.get("minimum_similarity", 0.75)) self.formatting = settings.get( "formatting", - "{newline}{quote_begin}> {quote} {quote_end}[{page}]{note_begin}{newline}Note: {note} {note_end} #{tag}", + "{%quote_container> {quote} %}[{page}]{%note_container{newline}Note: {note} %}{%tag_container #{tag}%}", ) self.color_mapping = settings.get("color_mapping", {}) @@ -350,7 +350,9 @@ class ExtractPlugin(PapersPlugin): """ existing = read_text_file(notepath) # removed annotations already found in the note - existing_dropped = [x for x in annotations if x.formatted(self.formatting) not in existing] + existing_dropped = [ + x for x in annotations if x.formatted(self.formatting) not in existing + ] if not existing_dropped: return From d0027ca5d1e1e77ee610da171f91525b56bda3ba Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sun, 25 Dec 2022 00:18:11 +0100 Subject: [PATCH 6/9] Add configurable minimum color similarity --- extract/extract.py | 48 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/extract/extract.py b/extract/extract.py index 9d4875b..a32be32 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -18,6 +18,8 @@ from pubs.content import check_file, read_text_file, write_file from pubs.query import get_paper_filter CONFIRMATION_PAPER_THRESHOLD = 5 +TEXT_SIMILARITY_MINIMUM = 0.75 +COLOR_SIMILARITY_MINIMUM = 0.833 COLORS = { "red": (1, 0, 0), @@ -42,7 +44,13 @@ class Annotation: colors: Dict = field(default_factory=lambda: {"stroke": (0.0, 0.0, 0.0)}) tag: str = "" - def formatted(self, formatting): + def format(self, formatting): + """Return a formatted string of the annotation. + + Given a provided formatting pattern, this method returns the annotation + formatted with the correct marker replacements and removals, ready + for display or writing. + """ output = formatting replacements = { r"{quote}": self.text, @@ -67,18 +75,31 @@ class Annotation: @property def colorname(self): + """Return the stringified version of the annotation color. + + Finds the closest named color to the annotation and returns it. + """ annot_colors = ( self.colors.get("stroke") or self.colors.get("fill") or (0.0, 0.0, 0.0) ) nearest = None - smallest_dist = 2.0 + minimum_similarity = COLOR_SIMILARITY_THRESHOLD for name, values in COLORS.items(): - dist = math.dist([*values], [*annot_colors]) - if dist < smallest_dist: - smallest_dist = dist + similarity_ratio = self._color_similarity_ratio(values, annot_colors) + if similarity_ratio > minimum_similarity: + minimum_similarity = similarity_ratio nearest = name return nearest + def _color_similarity_ratio(self, color_one, color_two): + """Return the similarity of two colors between 0 and 1. + + Takes two rgb color tuples made of floats between 0 and 1, e.g. (1, 0.65, 0) for orange, + and returns the similarity between them, with 1 being the same color and 0 being the + difference between full black and full white, as a float. + """ + return 1 - (abs(math.dist([*color_one], [*color_two])) / 3) + class ExtractPlugin(PapersPlugin): """Extract annotations from any pdf document. @@ -105,7 +126,12 @@ class ExtractPlugin(PapersPlugin): settings = conf["plugins"].get("extract", {}) self.on_import = settings.get("on_import", False) - self.minimum_similarity = float(settings.get("minimum_similarity", 0.75)) + self.minimum_similarity = float( + settings.get("minimum_similarity", TEXT_SIMILARITY_MINIMUM) + ) + self.minimum_color_similarity = float( + settings.get("minimum_color_similarity", COLOR_SIMILARITY_MINIMUM) + ) self.formatting = settings.get( "formatting", "{%quote_container> {quote} %}[{page}]{%note_container{newline}Note: {note} %}{%tag_container #{tag}%}", @@ -193,7 +219,7 @@ class ExtractPlugin(PapersPlugin): return papers_annotated def tag_from_colorname(self, colorname): - return self.color_mapping.get(colorname) + return self.color_mapping.get(colorname, "") def _gather_papers(self, conf, args): """Get all papers for citekeys. @@ -307,7 +333,7 @@ class ExtractPlugin(PapersPlugin): output += f"------ {citekey} ------\n" for annotation in annotations: # for annot in annotations: - output += f"{annotation.formatted(self.formatting)}\n" + output += f"{annotation.format(self.formatting)}\n" output += "\n" self.ui.message(output) @@ -339,7 +365,7 @@ class ExtractPlugin(PapersPlugin): """ output = "# Annotations\n\n" for annotation in annotations: - output += f"{annotation.formatted(self.formatting)}\n\n" + output += f"{annotation.format(self.formatting)}\n\n" write_file(notepath, output, "w") def _append_to_note(self, notepath, annotations): @@ -351,14 +377,14 @@ class ExtractPlugin(PapersPlugin): existing = read_text_file(notepath) # removed annotations already found in the note existing_dropped = [ - x for x in annotations if x.formatted(self.formatting) not in existing + x for x in annotations if x.format(self.formatting) not in existing ] if not existing_dropped: return output = "" for annotation in existing_dropped: - output += f"{annotation.formatted(self.formatting)}\n\n" + output += f"{annotation.format(self.formatting)}\n\n" write_file(notepath, output, "a") From 1ab43a2a0ebc4126ba105c9cb93b92e910dbfeeb Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sun, 25 Dec 2022 00:31:35 +0100 Subject: [PATCH 7/9] Add instructions for similarity and tag configuration --- README.md | 93 ++++++++++++++++++++++++++++------------------ extract/extract.py | 2 +- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 7fda631..18702f7 100644 --- a/README.md +++ b/README.md @@ -20,40 +20,17 @@ active = extract [[extract]] on_import = False -formatting = "[{page}]{newline}{quote_begin}> {quote}{newline}{quote_end}{note_begin}Note: {note}{note_end}" -minimum_similarity = 0.75 +minimum_text_similarity = 0.75 +minimum_color_similarity = 0.833 +formatting = "{%quote_container> {quote} %}[{page}]{%note_container{newline}Note: {note} %}{%tag_container #{tag}%}" ``` If `on_import` is `True` extraction is automatically run whenever a new document is added to the library, if false extraction has to be handled manually. -`formatting` takes a string with a variety of template options. You can use any of the following: +--- -- `{page}`: The page number the annotation was found on. -- `{quote}`: The actual quoted string (i.e. highlighted). -- `{note}`: The annotation note (i.e. addded reader). -- `{quote_begin}`: Mark the area that begins a quotation. Useful to get rid of prefixes or suffixes if no quotation exists. Requires a corresponding `{quote_end}` to be set after. -- `{note_begin}`: Same idea for the note area. Requires a corresponding `{note_end}` to be set after. - -For example, the default formatting string will result in this output: - -```markdown -[4] -> came “urban rights” caused collective outrage. Thus, through- out 2007 and 2008, protestors in towns and villages across the Nile Delta poured into the streets to rally over cuts in water flow. Deployment of massive riot police could not stop -Note: Often illegally connected to network, ‘revolution of the thirsty’ -``` - -The formatting string `{quote_begin}Quote: {quote}{page}{newline}{quote_end}{note_begin}Note: {note}{note_end}` -will result in the following output: - -```markdown -Quote: came “urban rights” caused collective outrage. Thus, through- out 2007 and 2008, protestors in towns and villages across the Nile Delta poured into the streets to rally over cuts in water flow. Deployment of massive riot police could not stop [4] -Note: Often illegally connected to network, ‘revolution of the thirsty’ -``` - -Note, however, that in this example the page number if *only* displayed if the annotation contains a quote. - -`minimum_similarity` sets the required similarity of an annotation's note and written words to be viewed +`minimum_text_similarity` sets the required similarity of an annotation's note and written words to be viewed as one. Any annotation that has both and is *under* the minimum similarity will be added in the following form: ```markdown @@ -64,6 +41,47 @@ Note: my additional thoughts That is, the extractor detects additional written words by whoever annotated and adds them to the extraction. The option generally should not take too much tuning, but it is there if you need it. +--- + +`minimum_color_similarity` sets the required similarity of highlight/annotation colors to be recognized as the 'pure' versions of themselves for color mapping (see below). With a low required similarity, for example dark green and light green will both be recognized simply as 'green' while a high similarity will not match them, instead only matching closer matches to a pure (0, 255, 0) green value. + +This should generally be an alright default but is here to be changed for example if you work with a lot of different annotation colors (where dark purple and light purple get different meanings) and get false positives. + +--- + +The plugin contains a configuration sub-category of `color_mapping`: Here, you can define meaning for your highlight/annotation colors. For example, if you always highlight the main arguments and findings in orange and always highlight things you have to follow up on in blue, you can assign the meanings 'important' and 'todo' to them respectively as follows: + +```ini +[[[color_mapping]]] +orange = "important" +blue = "todo" +``` + +Currently recognized colors are: `red` `green` `blue` `yellow` `purple` `orange`. +Since these meanings are often highly dependent on personal organization and reading systems, +no defaults are set here. + +--- + +`formatting` takes a string with a variety of template options. You can use any of the following: + +- `{page}`: The page number the annotation was found on. +- `{quote}`: The actual quoted string (i.e. highlighted). +- `{note}`: The annotation note (i.e. addded reader). +- `{%quote_container [other text] %}`: Mark the area that contains a quotation. Useful to get rid of prefixes or suffixes if no quotation exists. Usually contains some plain text and a `{quote}` element. Can *not* be nested with other containers. +- `{%note_container [other text] %}`: Mark the area that contains a note. Useful to get rid of prefixes or suffixes if no note exists. Usually contains some plain text and a `{note}` element. Can *not* be nested with other containers. +- `{%tag_container [other text] %}`: Mark the area that contains a tag. Useful to get rid of prefixes or suffixes if no tag exists. Usually contains some plain text and a `{tag}` element. Can *not* be nested with other containers. +- `{newline}`: Add a line break in the resulting annotation display. + +For example, the default formatting string `"{%quote_container> {quote} %}[{page}]{%note_container{newline}Note: {note} %}{%tag_container #{tag}%}"` will result in this output: + +``` +> Mobilizing the TPSN scheme (see above) and drawing on cultural political economy and critical governance studies, this landmark article offers an alternative account [5] +Note: a really intersting take on polydolywhopp +``` + +Container marks are useful to encapsulate a specific type of the annotation, so extracted annotations in the end don't contains useless linebreaks or quotations markup. + ## Usage: `pubs extract [-h|-w|-e] ` @@ -131,20 +149,21 @@ content, because then we can just use that. It is harder to parse if it does not ## Roadmap: - [x] extracts highlights and annotations from a doc file (e.g. using PyMuPDF) -- [ ] puts those in the annotation file of a doc in a customizable format +- [x] puts those in the annotation file of a doc in a customizable format - [x] option to have it automatically run after a file is added? - option to have it run whenever a pdf in the library was updated? - [ ] needs some way to delimit where it puts stuff and user stuff is in note - [ ] one way is to have it look at `> [17] here be extracted annotation from page seventeen` annotations and put it in between - [x] another, probably simpler first, is to just append missing annotations to the end of the note -- [ ] some highlights (or annotations in general) do not contain text as content - - [ ] pymupdf can extract the content of the underlying rectangle (mostly) - - [ ] issue is that sometimes the highlight contents are in content, sometimes a user comment instead - - [ ] we could have a comparison function which estimates how 'close' the two text snippets are and act accordingly -- [ ] config option to map colors in annotations to meaning ('read', 'important', 'extra') in pubs - - [ ] colors are given in very exact 0.6509979 RGB values, meaning we could once again estimate if a color is 'close enough' in distance to tag it accordingly -- [ ] make invoking the command run a query if corresponding option provided (or whatever) in pubs syntax and use resulting papers - - [ ] confirm for many papers? +- [x] some highlights (or annotations in general) do not contain text as content + - [x] pymupdf can extract the content of the underlying rectangle (mostly) + - [x] issue is that sometimes the highlight contents are in content, sometimes a user comment instead + - [x] we could have a comparison function which estimates how 'close' the two text snippets are and act accordingly -> using levenshtein distance +- [x] config option to map colors in annotations to meaning ('read', 'important', 'extra') in pubs + - [x] colors are given in very exact 0.6509979 RGB values, meaning we could once again estimate if a color is 'close enough' in distance to tag it accordingly -> using euclidian distance + - [ ] support custom colors by setting a float tuple in configuration +- [x] make invoking the command run a query if corresponding option provided (or whatever) in pubs syntax and use resulting papers + - [x] confirm for many papers? - [ ] warning when the amount of annotations in file is different than the amount extracted? ## Things that would also be nice in pubs in general and don't really belong in this repository diff --git a/extract/extract.py b/extract/extract.py index a32be32..676ce46 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -127,7 +127,7 @@ class ExtractPlugin(PapersPlugin): settings = conf["plugins"].get("extract", {}) self.on_import = settings.get("on_import", False) self.minimum_similarity = float( - settings.get("minimum_similarity", TEXT_SIMILARITY_MINIMUM) + settings.get("minimum_text_similarity", TEXT_SIMILARITY_MINIMUM) ) self.minimum_color_similarity = float( settings.get("minimum_color_similarity", COLOR_SIMILARITY_MINIMUM) From ac711ce1086c9b375994e3171a11565d91422374 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sun, 25 Dec 2022 00:33:08 +0100 Subject: [PATCH 8/9] Change naming of color_mapping to tag configuration --- README.md | 4 ++-- extract/extract.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 18702f7..cf68cfa 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,10 @@ This should generally be an alright default but is here to be changed for exampl --- -The plugin contains a configuration sub-category of `color_mapping`: Here, you can define meaning for your highlight/annotation colors. For example, if you always highlight the main arguments and findings in orange and always highlight things you have to follow up on in blue, you can assign the meanings 'important' and 'todo' to them respectively as follows: +The plugin contains a configuration sub-category of `tags`: Here, you can define meaning for your highlight/annotation colors. For example, if you always highlight the main arguments and findings in orange and always highlight things you have to follow up on in blue, you can assign the meanings 'important' and 'todo' to them respectively as follows: ```ini -[[[color_mapping]]] +[[[tags]]] orange = "important" blue = "todo" ``` diff --git a/extract/extract.py b/extract/extract.py index 676ce46..47e9d36 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -136,7 +136,7 @@ class ExtractPlugin(PapersPlugin): "formatting", "{%quote_container> {quote} %}[{page}]{%note_container{newline}Note: {note} %}{%tag_container #{tag}%}", ) - self.color_mapping = settings.get("color_mapping", {}) + self.color_mapping = settings.get("tags", {}) def update_parser(self, subparsers, _): """Allow the usage of the pubs extract subcommand""" From 274bb47c64de91a23678e8233fffcf0d9e782613 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sun, 25 Dec 2022 00:34:27 +0100 Subject: [PATCH 9/9] Fix variable naming --- extract/extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract/extract.py b/extract/extract.py index 47e9d36..2d107e6 100644 --- a/extract/extract.py +++ b/extract/extract.py @@ -83,7 +83,7 @@ class Annotation: self.colors.get("stroke") or self.colors.get("fill") or (0.0, 0.0, 0.0) ) nearest = None - minimum_similarity = COLOR_SIMILARITY_THRESHOLD + minimum_similarity = COLOR_SIMILARITY_MINIMUM for name, values in COLORS.items(): similarity_ratio = self._color_similarity_ratio(values, annot_colors) if similarity_ratio > minimum_similarity: