Fix: clean data accepted key

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Fix: Refactor and organize modules (@lwasser, #76)
 - Fix: Rename and organize `clean.py` module into `utils_parse` and `utils_clean` (@lwasser, @willingc, #121)
 - Fix: Add tests for all utils functions (@lwasser, #122)
+- Fix: Bug where date_accepted is removed (@lwasser, #129)
 
 
 ## [v0.2.3] - 2024-02-29

src/pyosmeta/cli/process_reviews.py

@@ -32,8 +32,6 @@ def main():
     # date_accepted_(month/day/year)
     # Get all issues for approved packages - load as dict
     issues = process_review.return_response()
-    # TODO: this method parse_issue_header is working but the parsing code is
-    # hard to follow
     accepted_reviews = process_review.parse_issue_header(issues, 45)
 
     # Update gh metrics via api for all packages
@@ -48,6 +46,7 @@ def main():
         # First add gh meta to each dict
         print("Parsing & validating", key)
         try:
+
             final_reviews[key] = ReviewModel(**review)
         except ValidationError as ve:
             print(key, ":", ve)

src/pyosmeta/models.py

@@ -220,9 +220,7 @@ class ReviewModel(BaseModel):
     version_accepted: str | None = None
     date_accepted: str | None = Field(
         default=None,
-        validation_alias=AliasChoices(
-            "date_accepted_(month/day/year)", "Date accepted"
-        ),
+        validation_alias=AliasChoices("Date accepted", "date_accepted"),
     )
     created_at: str = None
     updated_at: str = None

src/pyosmeta/parse_issues.py

@@ -6,7 +6,7 @@
 from typing import Any
 
 from .contributors import ProcessContributors
-from .utils_clean import clean_markdown
+from .utils_clean import clean_date_accepted_key, clean_markdown
 from .utils_parse import parse_user_names
 
 
@@ -150,6 +150,10 @@ def parse_issue_header(
                 continue
 
             review[pkg_name] = self.get_issue_meta(body_data, total_lines)
+
+            # Normalize date accepted key to be the same in each review
+            review[pkg_name] = clean_date_accepted_key(review[pkg_name])
+
             # Add issue open and close date to package meta from GH response
             # Date cleaning happens via pydantic validator not here
             for a_date in meta_dates:
@@ -191,7 +195,7 @@ def parse_issue_header(
                 "archive",
                 "version_accepted",
                 "joss_doi",
-                "date_accepted_(month/day/year)",
+                "date_accepted",
                 "categories",
                 "partners",
                 "domain",

src/pyosmeta/utils_clean.py

@@ -6,6 +6,8 @@
 import re
 from datetime import datetime
 
+from typing import Any
+
 
 def clean_date(source_date: str | None) -> datetime | str:
     """Cleans up a date string to a consistent datetime format.
@@ -61,3 +63,33 @@ def clean_markdown(source_text: str) -> str:
     cleaned = re.sub(pattern, "", source_text)
 
     return cleaned
+
+
+def clean_date_accepted_key(review_dict: dict[str, Any]) -> dict[str, str]:
+    """
+    Normalize date_accepted keys in our review dictionary.
+
+    In our reviews we have various templates that have evolved over the past
+    5 years (since 2019). Some used date accepted, some have date accepted
+    (month/day/year) and some have month-day-year. Rather than try to
+    account for all of these this is a helper that simply updates the key
+    to be date_accepted regardless of what is found after that text.
+
+    Parameters
+    ----------
+    review_dict : dict
+        Dictionary containing date_accepted key and other review data.
+
+    Returns
+    -------
+    dict
+        The modified review dictionary with normalized date_accepted key.
+    """
+    for key in list(review_dict.keys()):
+        if key.startswith("date_accepted"):
+            # Remove the old key
+            value = review_dict.pop(key)
+            # Add the new key with the value
+            review_dict["date_accepted"] = value
+            break
+    return review_dict

tests/unit/test_review_model.py → tests/integration/test_review_model.py

@@ -18,7 +18,7 @@
     "archive": "[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.8384174.svg)](https://doi.org/10.5281/zenodo.8384174)",
     "version_accepted": "5.1.1",
     "joss_doi": "[![DOI](https://joss.theoj.org/papers/10.21105/joss.01832/status.svg)](https://joss.theoj.org/papers/10.21105/joss.01832)",
-    "date_accepted_(month/day/year)": "01/18/2024",
+    "date_accepted": "01/18/2024",
     "categories": [
         "data-retrieval",
         "data-extraction",

tests/unit/test_utils_clean.py

@@ -2,7 +2,12 @@
 
 import pytest
 
-from pyosmeta.utils_clean import clean_date, clean_markdown, clean_name
+from pyosmeta.utils_clean import (
+    clean_date,
+    clean_date_accepted_key,
+    clean_markdown,
+    clean_name,
+)
 
 
 @pytest.mark.parametrize(
@@ -55,3 +60,27 @@ def test_clean_markdown(input_string, expected_output):
 )
 def test_clean_name(input_name, expected_output):
     assert clean_name(input_name) == expected_output
+
+
+@pytest.mark.parametrize(
+    "input_dict, expected_output",
+    [
+        # Test case where key starts with "date_accepted"
+        (
+            {"date_accepted_(month-day-year)": "2024-03-07"},
+            {"date_accepted": "2024-03-07"},
+        ),
+        (
+            {"date_accepted_(month/day/year)": "2024/03/07"},
+            {"date_accepted": "2024/03/07"},
+        ),
+        # Test where key does not start with "date_accepted"
+        ({"other_key": "value"}, {"other_key": "value"}),
+        (
+            {"date_accepted": "2024-03-07"},
+            {"date_accepted": "2024-03-07"},
+        ),
+    ],
+)
+def test_clean_date_accepted_key(input_dict, expected_output):
+    assert clean_date_accepted_key(input_dict) == expected_output