From 7684a0370a9a70b0ffd6f4ea0114c5e3c40541da Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 14 Jan 2023 11:53:01 +0100 Subject: [PATCH 1/4] Inject secrets into ci pipeline --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 6c3c118..94ecb02 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -52,6 +52,7 @@ pipeline: - poetry install - echo "----------------- publishing to pypi ------------------" - poetry publish --username "$PYPI_USERNAME" --password "$PYPI_PASSWORD" + secrets: [ pypi_username, pypi_password ] when: event: tag tag: v* From 836079cecbbfc27dd63a40311aa4331821528440 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 14 Jan 2023 11:34:35 +0100 Subject: [PATCH 2/4] Remove roadmap from README, add plugin status note --- README.md | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8432437..61f32bb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # pubs-extract +[![status-badge](https://ci.martyoeh.me/api/badges/Marty/pubs-extract/status.svg)](https://ci.martyoeh.me/Marty/pubs-extract) + Quickly extract annotations from your pdf files with the help of the pubs bibliography manager. +Easily organize your highlights and thoughts next to your documents. ## Installation: @@ -19,6 +22,9 @@ active = extract To check if everything is working you can do `pubs --help` which should show you the new extract command. You will be set up with the default options but if you want to change anything, read on in configuration below. +> **Note** +> This plugin is in fairly early development. It does what I need it to do, but if you have a meticulously organized library *please* make backups before doing any operation on your notes, or make use of the pubs-included git plugin. + ## Configuration: In your pubs configuration file: @@ -150,7 +156,7 @@ Pull requests tackling one of these areas of course very welcome. ## Issues -A note on the extraction. Highlights in pdfs can be somewhat difficult to parse +A note on the extraction: Highlights in pdfs can be somewhat difficult to parse (as are most things in them). Sometimes they contain the selected text that is written on the page, sometimes they contain the annotators thoughts as a note, sometimes they contain nothing. This plugin makes an effort to find the right combination and extract the written words, @@ -164,25 +170,12 @@ or even cut a few off. I am not sure if there is much I can do about this. -## Roadmap: +--- -- [x] extracts highlights and annotations from a doc file (e.g. using PyMuPDF) -- [x] puts those in the annotation file of a doc in a customizable format -- [x] option to have it automatically run after a file is added? - - option to have it run whenever a pdf in the library was updated? -- [ ] needs some way to delimit where it puts stuff and user stuff is in note - - [ ] one way is to have it look at `> [17] here be extracted annotation from page seventeen` annotations and put it in between - - [x] another, probably simpler first, is to just append missing annotations to the end of the note - - [ ] use similarity search instead of literal search for existing annotation (levenshtein)? -- [x] some highlights (or annotations in general) do not contain text as content - - [x] pymupdf can extract the content of the underlying rectangle (mostly) - - [x] issue is that sometimes the highlight contents are in content, sometimes a user comment instead - - [x] we could have a comparison function which estimates how 'close' the two text snippets are and act accordingly -> using levenshtein distance - - [ ] sometimes the underyling rectangle is empty too, what to do then? discard annotation? -- [x] config option to map colors in annotations to meaning ('read', 'important', 'extra') in pubs - - [x] colors are given in very exact 0.6509979 RGB values, meaning we could once again estimate if a color is 'close enough' in distance to tag it accordingly -> using euclidian distance - - [ ] support custom colors by setting a float tuple in configuration -- [x] make invoking the command run a query if corresponding option provided (or whatever) in pubs syntax and use resulting papers - - [x] confirm for many papers? -- [ ] warning when the amount of annotations in file is different than the amount extracted? -- [ ] tests tests tests tests tests, lah-di-dah +If you spot a bug or have an idea feel free to open an issue.\ +I might be slow to respond but will consider them all! + +Pull requests are warmly welcomed.\ +If they are big changes or additions let's talk about them in an issue first. + +Thanks for using my software ❤️ From dcfd30645176ea3d4e8c733d6cd21bbe9165e52e Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 14 Jan 2023 11:39:43 +0100 Subject: [PATCH 3/4] Add distance calculation information to docstrings --- pubs/plugs/extract/annotation.py | 3 ++- pubs/plugs/extract/extract.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pubs/plugs/extract/annotation.py b/pubs/plugs/extract/annotation.py index 0baefbb..a12dec8 100644 --- a/pubs/plugs/extract/annotation.py +++ b/pubs/plugs/extract/annotation.py @@ -87,7 +87,8 @@ class Annotation: def colorname(self): """Return the stringified version of the annotation color. - Finds the closest named color to the annotation and returns it. + Finds the closest named color to the annotation and returns it, + using euclidian distance between the two color vectors. """ annot_colors = ( self.colors.get("stroke") or self.colors.get("fill") or (0.0, 0.0, 0.0) diff --git a/pubs/plugs/extract/extract.py b/pubs/plugs/extract/extract.py index c513813..e5c356d 100644 --- a/pubs/plugs/extract/extract.py +++ b/pubs/plugs/extract/extract.py @@ -225,7 +225,8 @@ class ExtractPlugin(PapersPlugin): that is only the written words, sometimes that is only annotation notes, sometimes it is both. Runs a similarity comparison between strings to find out whether they - should both be included or are doubling up. + should both be included or are doubling up, using + Levenshtein distance. """ content = annotation.info["content"].replace("\n", " ") written = page.get_textbox(annotation.rect).replace("\n", " ") From db9294d8f2417e161f24ec31a048c76c2555ea61 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Sat, 14 Jan 2023 11:39:55 +0100 Subject: [PATCH 4/4] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0ee52ce..41f3ad7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pubs-extract" -version = "0.1.0" +version = "0.2.0" description = "A pdf annotation extraction plugin for pubs bibliography manager" authors = ["Marty Oehme "] license = "LGPL-3.0"