Add more tests and GH action to run tests (#5)

* Add GH action for tests

* Move the assertion into main function

* recreate lock file to fix pytest deprecation warning

* migrate to pydantic v2

* add speakers example test

* remove redundant conversion

Author: egeakman

.github/workflows/tests.yaml

@@ -0,0 +1,25 @@
+name: Tests
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          cache: "pip"
+
+      - name: Install dependencies
+        run: make deps/pre install
+
+      - name: Run pytest
+        run: make test

Makefile

@@ -25,3 +25,6 @@ test:
 pre-commit:
 	pre-commit install
 	pre-commit run --all-files
+
+clean:
+	git clean -xdf

data/examples/output/speakers.json

@@ -5,7 +5,7 @@
     "biography": null,
     "avatar": "https://pretalx.com/media/avatars/picture.jpg",
     "slug": "a-speaker",
-    "submissions": [],
+    "submissions": ["A8CD3F"],
     "affiliation": "A Company",
     "homepage": null,
     "twitter": null,

requirements.in

@@ -1,6 +1,4 @@
 pdbpp
-black
-isort
 pytest
 pre-commit
 

requirements.txt

@@ -4,18 +4,16 @@
 #
 #    pip-compile
 #
-attrs==21.4.0
-    # via pytest
-black==22.3.0
-    # via -r requirements.in
-certifi==2021.10.8
+annotated-types==0.6.0
+    # via pydantic
+attrs==23.2.0
+    # via wmctrl
+certifi==2024.2.2
     # via requests
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==2.0.12
+charset-normalizer==3.3.2
     # via requests
-click==8.1.3
-    # via black
 distlib==0.3.8
     # via virtualenv
 fancycompleter==0.9.1
@@ -24,61 +22,51 @@ filelock==3.14.0
     # via virtualenv
 identify==2.5.36
     # via pre-commit
-idna==3.3
+idna==3.7
     # via requests
-iniconfig==1.1.1
+iniconfig==2.0.0
     # via pytest
-isort==5.10.1
-    # via -r requirements.in
-mypy-extensions==0.4.3
-    # via black
 nodeenv==1.8.0
     # via pre-commit
-packaging==21.3
+packaging==24.0
     # via pytest
-pathspec==0.9.0
-    # via black
 pdbpp==0.10.3
     # via -r requirements.in
-platformdirs==2.5.2
-    # via
-    #   black
-    #   virtualenv
-pluggy==1.0.0
+platformdirs==4.2.2
+    # via virtualenv
+pluggy==1.5.0
     # via pytest
 pre-commit==3.7.1
     # via -r requirements.in
-py==1.11.0
-    # via pytest
-pydantic==1.9.0
+pydantic==2.7.1
     # via -r requirements.in
-pygments==2.12.0
+pydantic-core==2.18.2
+    # via pydantic
+pygments==2.18.0
     # via pdbpp
-pyparsing==3.0.9
-    # via packaging
 pyrepl==0.9.0
     # via fancycompleter
-pytest==7.1.2
+pytest==8.2.1
     # via -r requirements.in
-python-slugify==6.1.2
+python-slugify==8.0.4
     # via -r requirements.in
 pyyaml==6.0.1
     # via pre-commit
-requests==2.27.1
+requests==2.31.0
     # via -r requirements.in
 text-unidecode==1.3
     # via python-slugify
-tomli==2.0.1
-    # via pytest
 tqdm==4.66.4
     # via -r requirements.in
-typing-extensions==4.2.0
-    # via pydantic
-urllib3==1.26.9
+typing-extensions==4.11.0
+    # via
+    #   pydantic
+    #   pydantic-core
+urllib3==2.2.1
     # via requests
-virtualenv==20.21.1
+virtualenv==20.26.2
     # via pre-commit
-wmctrl==0.4
+wmctrl==0.5
     # via pdbpp
 
 # The following packages are considered to be unsafe in a requirements file:

src/transform.py

@@ -1,8 +1,7 @@
 import json
 from datetime import datetime
 
-from pydantic import BaseModel, Field
-from pydantic.class_validators import root_validator
+from pydantic import BaseModel, Field, model_validator
 from slugify import slugify
 
 from src.config import Config
@@ -35,7 +34,8 @@ class PretalxAnswer(BaseModel):
     submission_id: str | None
     speaker_id: str | None
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def extract(cls, values):
         values["question_text"] = values["question"]["question"]["en"]
         values["answer_text"] = values["answer"]
@@ -60,11 +60,12 @@ class PretalxSpeaker(BaseModel):
     twitter: str | None = None
     mastodon: str | None = None
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def extract(cls, values):
         values["slug"] = slugify(values["name"])
 
-        answers = [PretalxAnswer.parse_obj(ans) for ans in values["answers"]]
+        answers = [PretalxAnswer.model_validate(ans) for ans in values["answers"]]
 
         for answer in answers:
             if answer.question_text == SpeakerQuestion.affiliation:
@@ -107,16 +108,16 @@ class PretalxSubmission(BaseModel):
     start: datetime | None = None
     end: datetime | None = None
 
-    # TODO: once we have schedule data then we can prefill those in the code
-    # here
+    # TODO: once we have schedule data then we can prefill those in the code here
     talks_in_parallel: list[str] | None = None
     talks_after: list[str] | None = None
     next_talk_code: str | None = None
     prev_talk_code: str | None = None
 
     website_url: str | None = None
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def extract(cls, values):
         # # SubmissionType and Track have localised names. For this project we
         # # only care about their english versions, so we can extract them here
@@ -132,7 +133,7 @@ def extract(cls, values):
 
         values["speakers"] = sorted([s["code"] for s in values["speakers"]])
 
-        answers = [PretalxAnswer.parse_obj(ans) for ans in values["answers"]]
+        answers = [PretalxAnswer.model_validate(ans) for ans in values["answers"]]
 
         for answer in answers:
             # TODO if we need any other questions
@@ -148,6 +149,10 @@ def extract(cls, values):
             if answer.question_text == SubmissionQuestion.level:
                 values["level"] = answer.answer_text.lower()
 
+        # Convert duration to string for model validation
+        if isinstance(values["duration"], int):
+            values["duration"] = str(values["duration"])
+
         slug = slugify(values["title"])
         values["slug"] = slug
         values["website_url"] = f"https://ep2024.europython.eu/session/{slug}"
@@ -173,11 +178,7 @@ def parse_submissions() -> list[PretalxSubmission]:
     """
     with open(Config.raw_path / "submissions_latest.json") as fd:
         js = json.load(fd)
-        subs = []
-        for item in js:
-            sub = PretalxSubmission.parse_obj(item)
-            subs.append(sub)
-
+        subs = [PretalxSubmission.model_validate(item) for item in js]
     return subs
 
 
@@ -187,11 +188,7 @@ def parse_speakers() -> list[PretalxSpeaker]:
     """
     with open(Config.raw_path / "speakers_latest.json") as fd:
         js = json.load(fd)
-        speakers = []
-        for item in js:
-            speaker = PretalxSpeaker.parse_obj(item)
-            speakers.append(speaker)
-
+        speakers = [PretalxSpeaker.model_validate(item) for item in js]
     return speakers
 
 
@@ -217,7 +214,7 @@ def save_publishable_sessions():
 
     publishable = publishable_submissions()
 
-    data = {k: v.dict() for k, v in publishable.items()}
+    data = {k: v.model_dump() for k, v in publishable.items()}
     with open(path, "w") as fd:
         json.dump(data, fd, indent=2)
 
@@ -228,16 +225,16 @@ def save_publishable_speakers():
     publishable = publishable_submissions()
     speakers = publishable_speakers(publishable.keys())
 
-    data = {k: v.dict() for k, v in speakers.items()}
+    data = {k: v.model_dump() for k, v in speakers.items()}
     with open(path, "w") as fd:
         json.dump(data, fd, indent=2)
 
 
-assert len(set(s.slug for s in publishable_submissions().values())) == len(
-    publishable_submissions()
-)
-
 if __name__ == "__main__":
+    print("Checking for duplicate slugs...")
+    assert len(set(s.slug for s in publishable_submissions().values())) == len(
+        publishable_submissions()
+    )
     print("Saving publishable data...")
     save_publishable_sessions()
     save_publishable_speakers()

tests/test_examples_are_up_to_date.py

@@ -1,6 +1,6 @@
 import json
 
-from src.transform import PretalxSubmission
+from src.transform import PretalxSpeaker, PretalxSubmission
 
 with open("./data/examples/pretalx/submissions.json") as fd:
     pretalx_submissions = json.load(fd)
@@ -13,9 +13,21 @@ def test_sessions_example():
     assert pretalx_submissions[0]["code"] == "A8CD3F"
     pretalx = pretalx_submissions[0]
 
-    transformed = PretalxSubmission.parse_obj(pretalx)
+    transformed = PretalxSubmission.model_validate(pretalx)
 
     with open("./data/examples/output/sessions.json") as fd:
         sessions = json.load(fd)
 
-    assert transformed.dict() == sessions["A8CD3F"]
+    assert transformed.model_dump() == sessions["A8CD3F"]
+
+
+def test_speakers_example():
+    assert pretalx_speakers[0]["code"] == "F3DC8A"
+    pretalx = pretalx_speakers[0]
+
+    transformed = PretalxSpeaker.model_validate(pretalx)
+
+    with open("./data/examples/output/speakers.json") as fd:
+        speakers = json.load(fd)
+
+    assert transformed.model_dump() == speakers["F3DC8A"]