v0.1.6: semantic_location: remove JSON field (#52)

.github/workflows/ci.yaml

@@ -11,10 +11,11 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest, windows-latest]
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
         exclude: [
           {platform: windows-latest, python-version: "3.9"},
-          {platform: windows-latest, python-version: "3.10"}
+          {platform: windows-latest, python-version: "3.10"},
+          {platform: windows-latest, python-version: "3.11"}
         ]
 
     runs-on: ${{ matrix.platform }}
@@ -31,10 +32,10 @@ jobs:
           pip install '.[testing]'
       - name: Run mypy
         run: |
-          mypy --install-types --non-interactive ./google_takeout_parser
+          mypy --install-types --non-interactive ./google_takeout_parser ./tests
       - name: Run pytest
         run: |
           pytest
       - name: Run flake8
         run: |
-          flake8 ./google_takeout_parser
+          flake8 ./google_takeout_parser ./tests

google_takeout_parser/models.py

@@ -22,7 +22,6 @@
 from dataclasses import dataclass
 
 from .common import Res
-from .log import logger
 
 Url = str
 
@@ -167,7 +166,7 @@ class PlaceVisit(BaseEvent):
     startTime: datetime
     endTime: datetime
     sourceInfoDeviceTag: Optional[int]
-    otherCandidateLocationsJSON: str
+    otherCandidateLocations: List[CandidateLocation]
     # TODO: parse these into an enum of some kind? may be prone to breaking due to new values from google though...
     placeConfidence: Optional[str]  # older semantic history (pre-2018 didn't have it)
     placeVisitType: Optional[str]
@@ -183,19 +182,6 @@ def dt(self) -> datetime:  # type: ignore[override]
     def key(self) -> Tuple[float, float, int, Optional[float]]:
         return self.lat, self.lng, int(self.startTime.timestamp()), self.visitConfidence
 
-    @property
-    def otherCandidateLocations(self) -> List[CandidateLocation]:
-        import json
-
-        loaded = json.loads(self.otherCandidateLocationsJSON)
-        if not isinstance(loaded, list):
-            logger.warning(
-                f"loading candidate locations: expected list, got {type(loaded)}, {loaded}"
-            )
-            return []
-
-        return [CandidateLocation.from_dict(x) for x in loaded]
-
 
 @dataclass
 class ChromeHistory(BaseEvent):

google_takeout_parser/parse_json.py

@@ -175,20 +175,25 @@ def _parse_semantic_location_history(p: Path) -> Iterator[Res[PlaceVisit]]:
             continue
         try:
             location_json = placeVisit["location"]
-            missing_location_key = _check_required_keys(location_json, _sem_required_location_keys)
+            missing_location_key = _check_required_keys(
+                location_json, _sem_required_location_keys
+            )
             if missing_location_key is not None:
                 # handle these fully defensively, since nothing at all we can do if it's missing these properties
-                logger.debug(f"CandidateLocation: {p}, no key '{missing_location_key}' in {location_json}")
+                logger.debug(
+                    f"CandidateLocation: {p}, no key '{missing_location_key}' in {location_json}"
+                )
                 continue
             location = CandidateLocation.from_dict(location_json)
             duration = placeVisit["duration"]
             yield PlaceVisit(
                 name=location.name,
                 address=location.address,
                 # separators=(",", ":") removes whitespace from json.dumps
-                otherCandidateLocationsJSON=json.dumps(
-                    placeVisit.get("otherCandidateLocations", []), separators=(",", ":")
-                ),
+                otherCandidateLocations=[
+                    CandidateLocation.from_dict(pv)
+                    for pv in placeVisit.get("otherCandidateLocations", [])
+                ],
                 sourceInfoDeviceTag=location.sourceInfoDeviceTag,
                 placeConfidence=placeVisit.get("placeConfidence"),
                 placeVisitImportance=placeVisit.get("placeVisitImportance"),

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = google_takeout_parser
-version = 0.1.5
+version = 0.1.6
 description = Parses data out of your Google Takeout (History, Activity, Youtube, Locations, etc...)
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -18,6 +18,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
 keywords = google data parsing
 
 [options]

tests/test_json.py

@@ -1,5 +1,4 @@
 import json
-import dataclasses
 import datetime
 from pathlib import Path
 from typing import Iterator, Any
@@ -37,13 +36,13 @@ def test_parse_activity_json(tmp_path_f: Path) -> None:
         description=None,
         titleUrl=None,
         subtitles=[
-            ("Computer programming", None),
-            ("Computer Science", None),
-            ("PostgreSQL", None),
-            ("Technology", None),
+            models.Subtitles("Computer programming", None),
+            models.Subtitles("Computer Science", None),
+            models.Subtitles("PostgreSQL", None),
+            models.Subtitles("Technology", None),
         ],
         locationInfos=[
-            (
+            models.LocationInfo(
                 "At this general area",
                 "https://www.google.com/maps/@?api=1&map_action=map&center=lat,lon&zoom=12",
                 "From your Location History",
@@ -98,7 +97,7 @@ def test_parse_app_installs(tmp_path_f: Path) -> None:
     ]
 
 
-def test_location_old(tmp_path_f) -> None:
+def test_location_old(tmp_path_f: Path) -> None:
     contents = '{"locations": [{"timestampMs": "1512947698030", "latitudeE7": 351324213, "longitudeE7": -1122434441, "accuracy": 10}]}'
     fp = tmp_path_f / "file"
     fp.write_text(contents)
@@ -191,12 +190,9 @@ def test_semantic_location_history(tmp_path_f: Path) -> None:
     fp = tmp_path_f / "file"
     fp.write_text(json.dumps(data))
     res = list(prj._parse_semantic_location_history(fp))
-    objbase = res[0]
-    assert not isinstance(objbase, Exception)
+    obj = res[0]
+    assert not isinstance(obj, Exception)
     # remove JSON, compare manually below
-    objd = dataclasses.asdict(objbase)
-    del objd["otherCandidateLocationsJSON"]
-    obj = models.PlaceVisit(**objd, otherCandidateLocationsJSON="{}")
     assert obj == models.PlaceVisit(
         lat=55.5555555,
         lng=-106.6666666,
@@ -213,22 +209,20 @@ def test_semantic_location_history(tmp_path_f: Path) -> None:
             2017, 12, 11, 1, 20, 6, 106000, tzinfo=datetime.timezone.utc
         ),
         sourceInfoDeviceTag=987654321,
-        otherCandidateLocationsJSON="{}",
         placeConfidence="MEDIUM_CONFIDENCE",
         placeVisitImportance="MAIN",
         placeVisitType="SINGLE_PLACE",
         visitConfidence=65.45,
         editConfirmationStatus="NOT_CONFIRMED",
+        otherCandidateLocations=[
+            models.CandidateLocation(
+                lat=42.3984239,
+                lng=-156.5656565,
+                name="name2",
+                address="address2",
+                locationConfidence=24.475897,
+                placeId="XPRK4E4P",
+                sourceInfoDeviceTag=None,
+            )
+        ],
     )
-
-    assert objbase.otherCandidateLocations == [
-        models.CandidateLocation(
-            lat=42.3984239,
-            lng=-156.5656565,
-            name="name2",
-            address="address2",
-            locationConfidence=24.475897,
-            placeId="XPRK4E4P",
-            sourceInfoDeviceTag=None,
-        )
-    ]

tests/test_split_html.py

@@ -33,7 +33,7 @@ def in_golang_dir() -> Generator[None, None, None]:
     "TEST_GOLANG_SPLIT" not in os.environ,
     reason="TEST_GOLANG_SPLIT not set, skipping test",
 )
-def test_split_html(in_golang_dir) -> None:
+def test_split_html(in_golang_dir: None) -> None:
     with tempfile.TemporaryDirectory() as temp_dir:
         subprocess.run(
             [

tests/test_urls.py

@@ -1,8 +1,9 @@
 import logging
+from pytest import LogCaptureFixture
 from google_takeout_parser.http_allowlist import _convert_to_https
 
 
-def test__convert_to_https(caplog) -> None:
+def test_convert_to_https(caplog: LogCaptureFixture) -> None:
     with caplog.at_level(logging.DEBUG):
         url = "http://www.google.com"
         assert _convert_to_https(url) == "https://www.google.com"
@@ -22,7 +23,7 @@ def test__convert_to_https(caplog) -> None:
         url = "http://m.youtube.com/watch?v=123"
         assert _convert_to_https(url) == "https://m.youtube.com/watch?v=123"
 
-        from logzero import logger
+        from logzero import logger  # type: ignore[import]
 
         logger.propagate = True