Add all simultaneous activities to loop

While in nomie, multiple occurences of the same activity can be recorded
in one single event (e.g. you smoked multiple cigarettes and now record
that fact), Loop has no such concept.

Thus, we work around it by fudging the duplicate timestamps ever so
slightly (one millisecond each) and then adding them to the database.

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project tries to adhere to [Semantic Versioning](https://semver.org/spe
 
 ## [Unreleased]
 
+### Added 
+
+* Added import of duplicate activities
+  In nomie, you can have multiple activity tags in a single event,
+  e.g. #smoking #smoking. This does not work in Loop, but we import
+  them as separate counter instances instead.
+
+### Fixed
+
+* Create missing PRAGMA values for Loop SQLite database, fixing failing import
+
 ## [0.3.1] - 2021-09-30
 
 ### Fixed

README.md

@@ -23,6 +23,44 @@ poetry install
 poetry run habitmove <nomie-json>
 ```
 
+## Roadmap
+
+* [x] transform nomie data into dataclasses
+    * [x] Trackers
+    * [x] Events
+    * [x] Activities
+* [x] transform loop data into dataclasses
+    * [x] create habit
+    * [x] create some way of getting 'length' of habit (how many fields)
+    * [x] move extracting of 'activity' from loop to nomie functionality (coupled with nomie note representation)
+    * [x] make loop migration use this activity data
+    * [x] create repetition
+* [ ] clean up README
+* [ ] begin adding tests
+    * [ ] add some unit tests for various functions
+    * [ ] and at least an integration test for the stable database (loop-2021-12-02.db or equivalent)
+* [ ] abstract migration target away from loop
+* [ ] abstract import away from nomie
+* [ ] abstract importer/migrator themselves to work with other targets
+* [ ] allow migration to/from nomie/loop
+
+## Internal concepts
+
+The migrator has the concept of a tracker, an event, and an activity:
+
+* A tracker is the system / the goal / the accumulator that tracks a certain number of you 'doing' things. 
+    * In Loop Habit Tracker it corresponds to a Habit
+* An event is something happening, something being emitted but generally in its raw form. They do *not* 
+  exist for pure numerical value tracking.
+    * Nomie has the idea of an event as a little text note that you write
+    * Loop Habit Tracker has no equivalent of an event and they will simply be discarded
+    * Generally, this will be most useful if you wish to preserve and keep prose notes in the system
+* An activity is the interpreted variation of an event and which only effects your trackers.
+  Essentially something that changes a tracker's counter / value, like you doing sports or drinking water.
+    * Nomie keeps activities within its notes (as tags) and has no separate concept of them
+    * Loop Habit Tracker is only geared toward tracking activities and they are its main building block
+    * Generally, this will be quantifiable data and most useful for statistics and similar analysis
+
 # Schemas
 
 A collection of the schemas of Loop habit sqlite database for later migration

habitmove/loopdata.py

@@ -7,22 +7,22 @@ from uuid import uuid4
 @dataclass
 class Habit:
     name: str
-    type: Optional[int] = 0  # 1 is range counter
+    type: int = 0  # 1 is range counter
-    archived: Optional[int] = 0
+    archived: int = 0
-    color: Optional[int] = 0
+    color: int = 0
-    highlight: Optional[int] = 0
+    highlight: int = 0
-    freq_den: Optional[int] = 1
+    freq_den: int = 1
-    freq_num: Optional[int] = 1
+    freq_num: int = 1
-    target_value: Optional[int] = 0
+    target_value: int = 0
-    description: Optional[str] = ""
+    description: str = ""
-    question: Optional[str] = ""
+    question: str = ""
-    unit: Optional[str] = ""
+    unit: str = ""
-    position: Optional[int] = 0
+    position: int = 0
-    uuid: Optional[str] = None
+    uuid: str = ""
 
     # TODO test post init uuid setting
     def __post_init__(self):
-        if not self.uuid:
+        if not self.uuid or self.uuid == "":
             self.uuid = uuid4().hex
 
 

habitmove/nomiedata.py

@@ -17,19 +17,19 @@ class Tracker:
     uom: str
     # TODO no idea what include does
     include: str
-    min: Optional[int] = 0
+    min: int = 0
-    max: Optional[int] = 0
+    max: int = 0
-    goal: Optional[int] = 0
+    goal: int = 0
-    default: Optional[int] = 0
+    default: int = 0
     # TODO score can be string (if custom) or int (if simple good/bad)
-    score: Optional[str] = ""
+    score: str = ""
-    score_calc: Optional[list[dict[str, Any]]] = field(default_factory=lambda: [])
+    score_calc: list[dict[str, Any]] = field(default_factory=lambda: [])
 
 
 @dataclass(frozen=True)
 class Activity:
     tracker: Tracker
-    value: Optional[int] = 1
+    value: int = 1
 
 
 # A nomie note. Records any circumstance of 'something happened' through prose.
@@ -41,14 +41,14 @@ class Event:
     start: int
     end: int
     text: str
-    activities: Optional[list[Activity]] = field(default_factory=lambda: [])
+    activities: list[Activity] = field(default_factory=lambda: [])
-    score: Optional[int] = 0
+    score: int = 0
-    lat: Optional[float] = 0.0
+    lat: float = 0.0
-    lng: Optional[float] = 0.0
+    lng: float = 0.0
-    location: Optional[str] = ""
+    location: str = ""
-    modified: Optional[bool] = False
+    modified: bool = False
-    offset: Optional[str] = ""  # local timezone offset?
+    offset: str = ""  # local timezone offset?
-    source: Optional[str] = "n5"  # nomie version
+    source: str = "n5"  # nomie version
 
 
 @dataclass(frozen=True)

habitmove/repetitions.py

@@ -1,27 +1,33 @@
 import sqlite3
+from typing import Optional
 from datetime import datetime
 from habitmove.loopdata import Habit, Repetition
+from habitmove.nomiedata import Event
 
 
-def migrate(db, habitlist, events):
+def migrate(db: sqlite3.Connection, habits: list[Habit], events: list[Event]) -> None:
+    """Move Loop Activities contained in all Events matching Habits passed in into database.
+    :param db: Database to populate.
+    :param habits: List of Habits to find matching repetitions for.
+    :param events: List of events to find repetitions in.
+    """
     c = db.cursor()
-    habits = habit_list_add_ids(c, habitlist)
+    habits_with_sql_id = habit_list_add_ids(c, habits)
-    repetitions = get_all_repetitions(habits, events)
+    repetitions = get_all_repetitions(habits_with_sql_id, events)
     for rep in repetitions:
-        add_to_database(c, habits, rep)
+        add_to_database(c, habits_with_sql_id, rep)
 
 
 LOOP_RANGE_VALUE_MULTIPLIER = 1000
 
 
-def get_all_repetitions(habits, events):
+def get_all_repetitions(
-    """Return list of all repetitions found of habits in events passed in.
+    habits: dict[int, Habit], events: list[Event]
-    Parameters:
+) -> list[Repetition]:
-    habits (list): Collection of habits, with minimum necessary fields description and id.
+    """Return list of all repetitions found in events passed in.
-    events (list): Collection of events, with minimum necessary field end.
+    :param habits: Dict of habits with sql_ids that repetitions can be for.
-    Returns:
+    :param events: List of events to search through for repetitions.
-    repetitions (list): Collection of events transformed into Loop repetitions.
+    :return repetitions: List of Loop repetitions.
-                        Contains fields id, timestamp, value (for ranges).
     """
     repetitions = []
     for event in events:
@@ -40,29 +46,25 @@ def get_all_repetitions(habits, events):
 
 
 # TODO possibly just get rid of this entirely
-def habit_list_add_ids(c, habitlist):
+def habit_list_add_ids(c: sqlite3.Cursor, habitlist: list[Habit]) -> dict[int, Habit]:
     """Return the collection of habits with their sqlite id added.
-    Parameters:
+    :param c: SQL cursor of database to query.
-    c (sqlite.db.cursor): SQL cursor of database to query.
+    :param habitlist: Habits to get sql IDs for.
-    habitlist (list[Habit]): Full habit collection to return a simplified view of.
+    :return habit_id_dict: The habit collection as a dict with the keys
-    Returns:
-    habit_id_dict (dict[Habit]): The habit collection as a dict with the keys
                                  consisting of the habit's sqlite database ID.
     """
     with_id = {}
     for h in habitlist:
-        sql_id = fetch_habit_id(c, h.uuid)
+        sql_id = fetch_habit_id(c, h.uuid or "")
         with_id[sql_id] = h
 
     return with_id
 
 
-def fetch_habit_id(cursor: sqlite3.Cursor, uuid: str):
+def fetch_habit_id(cursor: sqlite3.Cursor, uuid: str) -> Optional[int]:
     """Return sqlite internal id for habit with uuid.
-    Parameters:
     :param c: SQL cursor of database to query.
     :param uuid: Unique id of habit to query for.
-    Returns:
     :return id: SQLite internal id for habit queried for.
     """
     cursor.execute("select id from Habits where uuid = ?", ([uuid]))
@@ -73,9 +75,8 @@ def fetch_habit_id(cursor: sqlite3.Cursor, uuid: str):
 
 def add_to_database(
     cursor: sqlite3.Cursor, habits: dict[int, Habit], repetition: Repetition
-):
+) -> None:
     """Insert the repetition into a sqlite3 table suitable for Loop.
-    Parameters:
     :param c: SQL cursor of database to query.
     :sql_id: Internal sqlite database id of the habit the repetition belongs to.
     """
@@ -95,3 +96,12 @@ def add_to_database(
                 print(
                     f"{sql_id}, {habit.name}: timestamp {datetime.fromtimestamp(repetition.timestamp/1000)} not unique, moving timestamp slightly."
                 )
+                add_to_database(
+                    cursor,
+                    habits,
+                    Repetition(
+                        repetition.habit_uuid,
+                        repetition.timestamp + 1,
+                        repetition.value,
+                    ),
+                )