Quality checks

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, E266, E501, W503
+max-complexity = 18
+select = B,C,E,F,W,T4,B9

.github/workflows/ci.yml

@@ -26,9 +26,9 @@ jobs:
     - name: Install dependencies
       run: |
         make update
-    - name: Test with pytest
+    - name: Run quality checks
       run: |
-        make test
+        make quality
     - name: Package
       run: |
         make docker_image

.github/workflows/package.yml

@@ -25,9 +25,9 @@ jobs:
     - name: Install dependencies
       run: |
         make update
-    - name: Test with pytest
+    - name: Run quality checks
       run: |
-        make test
+        make quality
     - name: Package
       run: |
         make docker_image

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+-   repo: local
+    hooks:
+    -   id: black
+        name: black
+        entry: black
+        language: python
+        types: [python]
+        additional_dependencies: [black]
+    -   id: flake8
+        name: flake8
+        entry: flake8
+        language: python
+        types: [python]
+        args: [--config=.flake8]
+        additional_dependencies: [flake8]

.vscode/settings.json

@@ -6,5 +6,10 @@
         "minio",
         "pymongo",
         "pytestmark"
-    ]
-}
+    ],
+    "python.linting.enabled": true,
+    "python.linting.flake8Enabled": true,
+    "python.linting.flake8Args": ["--config=.flake8"],
+    "editor.formatOnSave": true,
+    "python.formatting.provider": "black"
+}

makefile

@@ -48,4 +48,13 @@ mongodb:
 mongodb_bg:
 	docker compose -f mongodb-docker-compose.yml up -d
 stop_mongodb:
-	docker compose -f mongodb-docker-compose.yml down
+	docker compose -f mongodb-docker-compose.yml down
+
+quality:
+	@echo "Running code quality checks..."
+	flake8 src tests
+	black --check src tests
+	pre-commit run --all-files
+	@echo "Running tests..."
+	pytest -m "not integration"
+	@echo "Code quality checks completed."

requirements-dev.txt

@@ -1,4 +1,7 @@
 toml
 GitPython
 setuptools_scm>=8
-pytest
+pytest
+pre-commit
+black
+flake8

src/floridayvine/commands/floriday.py

@@ -9,20 +9,24 @@
 
 app = typer.Typer()
 
+
 @app.command()
 def floriday_connection_info():
     orgs = get_organizations()
     print("Connected to Floriday:")
     pprint(orgs)
 
+
 @app.command()
 def print_direct_sales():
     items = get_direct_sales()
     pprint(items)
 
+
 @app.command()
 def print_trade_items():
     trade_items = get_trade_items()
     pprint(trade_items)
 
+
 app.add_typer(sync.app, name="sync")

src/floridayvine/commands/minio.py

@@ -3,6 +3,7 @@
 
 app = typer.Typer()
 
+
 @app.command()
 def upload(
     ctx: typer.Context,

src/floridayvine/commands/sync.py

@@ -3,13 +3,15 @@
 
 app = typer.Typer()
 
+
 @app.command()
 def organizations(start_seq_number: int = 0, limit_result: int = 5):
     """
     Synchronize organizations data from Floriday.
     """
     sync_organizations(start_seq_number, limit_result)
 
+
 @app.command()
 def trade_items(start_seq_number: int = 0, limit_result: int = 5):
     """

src/floridayvine/commands/version.py

@@ -3,13 +3,16 @@
 
 app = typer.Typer()
 
+
 def get_version():
     return version("floridayvine")
 
+
 @app.command()
 def print_version():
     print(f"{get_version()}")
 
+
 @app.command()
 def about(ctx: typer.Context):
     print(

src/floridayvine/floriday/misc.py

@@ -1,5 +1,4 @@
 import os
-from pprint import pprint
 import time
 import requests
 from ..persistence import persist, get_max_sequence_number
@@ -9,7 +8,6 @@
     DirectSalesApi,
     OrganizationsApi,
 )
-from floriday_supplier_client.models.organization import Organization
 
 CLIENT_ID = os.getenv("FLORIDAY_CLIENT_ID")
 CLIENT_SECRET = os.getenv("FLORIDAY_CLIENT_SECRET")
@@ -78,7 +76,9 @@ def sync_entities(
         )
         max_seq_nr = sync_result.maximum_sequence_number
         for entity in sync_result.results:
-            print(f"Seq nr {entity.sequence_number}: Persisting {persist_entity(entity)} ...")
+            print(
+                f"Seq nr {entity.sequence_number}: Persisting {persist_entity(entity)} ..."
+            )
         my_sequence = sync_result.maximum_sequence_number
         time.sleep(0.5)
 
@@ -87,7 +87,7 @@ def sync_entities(
 
 def sync_organizations(start_seq_number=None, limit_result=5):
     api = OrganizationsApi(_clt)
-    
+
     def persist_org(org):
         persist("organizations", org.organization_id, org.to_dict())
         return org.name
@@ -117,7 +117,7 @@ def get_direct_sales():
 
 def sync_trade_items(start_seq_number=None, limit_result=5):
     api = TradeItemsApi(_clt)
-    
+
     def persist_item(item):
         persist("trade_items", item.trade_item_id, item.to_dict())
         return item.trade_item_name

src/floridayvine/persistence.py

@@ -1,10 +1,9 @@
 import os
 from pymongo import MongoClient
+import re
 
 mongodb_connection_string = os.getenv("MONGODB_CONNECTION_STRING")
 
-import re
-
 # Regular expression pattern to match and mask the password
 pattern = r"(mongodb://[^:]+:)([^@]+)(@.+)"
 masked_connection_string = re.sub(pattern, r"\1'*****'\3", mongodb_connection_string)

tests/test_upload_files_to_vine.py

@@ -19,9 +19,11 @@
 bucket = "floridayvine-testbucket"
 target_dir = "inbox"
 
+
 @pytest.fixture(scope="module", autouse=True)
 def setup():
-    assert minio_client.bucket_exists(bucket) == True
+    assert minio_client.bucket_exists(bucket) is True
+
 
 @pytest.fixture(autouse=True)
 def teardown_integration():
@@ -35,20 +37,25 @@ def teardown_integration():
         count_blobs() == 0
     ), f"Bucket '{bucket}' should be empty after each test, but it is not."
 
+
 def test_can_run_script_with_custom_bucket_and_target():
     source_dir = "tests/data/"
-    cp = subprocess.run(["floridayvine", "minio", "upload", source_dir, bucket, target_dir])
+    cp = subprocess.run(
+        ["floridayvine", "minio", "upload", source_dir, bucket, target_dir]
+    )
     assert cp.returncode == 0
 
     file_count = count_files(source_dir)
     blob_count = count_blobs(bucket)
 
     assert blob_count == file_count
 
+
 def count_blobs(in_bucket=bucket):
     blob_count = len(list(minio_client.list_objects(in_bucket, recursive=True)))
     return blob_count
 
+
 def count_files(source_dir):
     file_paths = glob.glob(f"{source_dir}/**", recursive=True)
     files_only = [f for f in file_paths if os.path.isfile(f)]

work/definition-of-done.md

@@ -1,14 +1,75 @@
 # Definition of Done
 
-The releasable artifacts are:
+This document outlines the specific criteria that must be met for a new releasable increment of our application to be considered complete. It serves as a checklist for developers and AI agents to ensure quality and consistency in our releases.
 
-* The code, zipped, downloadable.
-* A Docker container, available on <ghcr.io>.
+## Code Quality and Testing
 
-Code and documentation are pushed to the central code repository.
+- [ ] `make quality` command passes, which includes:
+  - Linting with `flake8` (as configured in .flake8)
+  - Formatting with `black`
+  - Running pre-commit hooks
+  - Running non-integration tests with `pytest`
+- [ ] All new functionality has corresponding unit tests in the `tests/` directory
+- [ ] No `TODO` comments remain in the code related to the current work
+- [ ] GitHub Actions CI workflow passes on the main branch
 
-Releases are identified using Semantic Versioning and tagged in the repository.
+## Documentation
 
-Environment variables are used for configuration options.
+- [ ] Docstrings are added/updated for all new/modified functions, classes, and modules
+- [ ] README.md is updated with any new features, dependencies, or usage instructions
+- [ ] Any new environment variables are added to both `.env.example` and `mongo/.env.default`
+- [ ] `docs/software_architecture.md` is updated if there are architectural changes
 
-CI job on Github action passes.
+## Functionality
+
+- [ ] All new CLI commands are implemented in `src/floridayvine/commands/` and properly integrated in `src/floridayvine/__main__.py`
+- [ ] New database operations are added to `src/floridayvine/persistence.py` and tested
+- [ ] MinIO operations in `src/floridayvine/minio.py` are updated and tested if affected
+- [ ] Floriday-related functionality in `src/floridayvine/floriday/` is updated and tested if affected
+
+## Performance and Security
+
+- [ ] No sensitive information (API keys, passwords) is hardcoded or logged
+- [ ] All database queries are optimized and indexed appropriately
+- [ ] Large data operations use appropriate batching or streaming techniques
+- [ ] Environment variables are used for all configuration options
+
+## DevOps and Deployment
+
+- [ ] `Dockerfile` and `docker-compose.yml` are updated if there are new dependencies or services
+- [ ] `requirements-dev.txt` is updated with any new development dependencies
+- [ ] Version number in `src/floridayvine/commands/version.py` is incremented appropriately
+- [ ] `floridayvine-crontab` is updated if there are changes to scheduled tasks
+- [ ] Configuration files (.flake8, .pre-commit-config.yaml, .vscode/settings.json) are up-to-date and committed to the repository
+
+## Cross-compatibility
+
+- [ ] Application runs successfully on both development and production environments
+- [ ] Any new dependencies are added to `pyproject.toml` and are compatible with both Linux and macOS
+
+## User Acceptance
+
+- [ ] New features or changes have been demonstrated and approved by the product owner
+
+## Release and Deployment
+
+- [ ] Code and documentation are pushed to the central code repository
+- [ ] Release is identified using Semantic Versioning
+- [ ] Release is tagged in the repository
+- [ ] GitHub Actions package workflow successfully builds and publishes the Docker container to ghcr.io
+
+## Final Checks
+
+- [ ] GitHub Actions CI workflow passes on the main branch, running all quality checks
+- [ ] `scripts/verify_credentials.sh` runs successfully with the updated code
+- [ ] Application can be built and run using `docker-compose up` without errors
+
+## Undone for now
+
+The following items are considered important for future releases but are not currently part of our Definition of Done:
+
+- [ ] Implement test coverage measurement and maintain or improve coverage (verify with coverage tool)
+- [ ] Create a script to automatically check if all environment variables in `.env.example` and `mongo/.env.default` are up to date
+- [ ] Implement a zipped version of the code for download as part of the release process
+
+This Definition of Done should be reviewed and updated periodically to reflect the evolving needs of the project. Items from the "Undone for now" section should be considered for inclusion in the main DoD as they become implemented and integrated into our development process.

work/dod-compliance-actions.md

@@ -0,0 +1,50 @@
+# Actions to Consider for DoD Compliance
+
+This document outlines actions to take to ensure compliance with the Definition of Done. These are steps to be taken during development and before considering a release complete.
+
+## Code Quality and Testing
+
+1. Review and add unit tests for any new functionality in the `tests/` directory.
+2. Search for and address any remaining `TODO` comments in the code.
+
+## Documentation
+
+3. Review and update docstrings for all functions, classes, and modules in the project.
+4. Update README.md with any new features, dependencies, or usage instructions.
+5. Add instructions for setting up and using pre-commit hooks to README.md.
+
+## Functionality
+
+6. Review `src/floridayvine/commands/` and `src/floridayvine/__main__.py` to ensure all CLI commands are properly implemented and integrated.
+7. Review `src/floridayvine/persistence.py` for any new database operations and ensure they are tested.
+8. Check `src/floridayvine/minio.py` for any updates and ensure related tests are in place.
+9. Review `src/floridayvine/floriday/` for any updates and ensure related tests are in place.
+
+## Performance and Security
+
+10. Conduct a security review to ensure no sensitive information is hardcoded or logged.
+11. Review and optimize database queries, ensuring proper indexing.
+12. Evaluate large data operations and implement batching or streaming techniques where necessary.
+
+## DevOps and Deployment
+
+13. Review `Dockerfile` and `docker-compose.yml` for any needed updates based on new dependencies or services.
+14. Update `requirements-dev.txt` with any new development dependencies.
+15. Increment the version number in `src/floridayvine/commands/version.py`.
+16. Review and update `floridayvine-crontab` if there are changes to scheduled tasks.
+
+## Cross-compatibility
+
+17. Test the application in both development and production environments.
+18. Review `pyproject.toml` for any new dependencies and ensure they are compatible with both Linux and macOS.
+
+## User Acceptance
+
+19. Schedule a demonstration of new features or changes with the product owner for approval.
+
+## Release Preparation
+
+20. Ensure all code and documentation are ready to be pushed to the central code repository.
+21. Prepare release notes documenting new features, bug fixes, and any breaking changes.
+
+These actions should help bring the project into compliance with the Definition of Done for the next release. Review and prioritize these actions as needed during the development process.