diff --git a/README.md b/README.md index c0f24a1..2079780 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,12 @@ [![PyPI version](https://badge.fury.io/py/django-typesense.svg)](https://pypi.python.org/pypi/django-typesense/) ![Python versions](https://img.shields.io/badge/python-%3E%3D3.8-brightgreen) ![Django Versions](https://img.shields.io/badge/django-%3E%3D3.2-brightgreen) -[![PyPI License](https://img.shields.io/pypi/l/django-typesense.svg)](https://pypi.python.org/pypi/django-typesense/) +[![PyPI License](https://img.shields.io/pypi/l/django-typesense.svg)](https://pypi.org/project/django-typesense/) -> [!WARNING] -> **This package is in the initial development phase. Do not use in production environment.** - ## What is it? Faster Django Admin powered by [Typesense](https://typesense.org/) - - ## Quick Start Guide ### Installation `pip install django-typesense` @@ -124,15 +119,54 @@ in the django app where the model you are creating a collection for is. > avoid triggering database queries that will negatively affect performance > [Issue #16](https://github.com/Siege-Software/django-typesense/issues/16). +Instead of this in the admin: +``` +@admin.display('Genre') +def genre_name(self, obj): + return obj.genre.name +``` + +Do this: + +``` +@admin.display('Genre') +def genre_name(self, obj): + # genre_name is field in the Collection. You can also store the object url as html + return obj.genre_name +``` + ### Update Collection Schema To add or remove fields to a collection's schema in place, update your collection then run: - `SongCollection().update_typesense_collection()` + `python manage.py updatecollections`. Consider adding this to your CI/CD pipeline. This also updates the [synonyms](#synonyms) + +### How updates are made to Typesense +1. Signals - +`django-typesense` listens to signal events (`post_save`, `pre_delete`, `m2m_changed`) to update typesense records. +If [`update_fields`](https://docs.djangoproject.com/en/4.2/ref/models/instances/#specifying-which-fields-to-save) +were provided in the save method, only these fields will be updated in typesense. + +2. Update query - +`django-typesense` overrides Django's `QuerySet.update` to make updates to typesense on the specified fields + +3. Manual - +You can also update typesense records manually e.g after doing a `bulk_create` +``` +objs = Song.objects.bulk_create( + [ + Song(title="Watch What I Do"), + Song(title="Midnight City"), + ] +) +collection = SongCollection(objs, many=True) +collection.update() +``` + ### Admin Integration To make a model admin display and search from the model's Typesense collection, the admin class should -inherit `TypesenseSearchAdminMixin` +inherit `TypesenseSearchAdminMixin`. This also adds Live Search to your admin changelist view. ``` from django_typesense.admin import TypesenseSearchAdminMixin @@ -211,6 +245,9 @@ class HasViewsFilter(admin.SimpleListFilter): return {} ``` +Note that simple lookups like the one above are done by default (hence no need to define `filter_by`) if +the `parameter_name` is a field in the collection + ### Synonyms The [synonyms](https://typesense.org/docs/0.25.1/api/synonyms.html) feature allows you to define search terms that should be considered equivalent. Synonyms should be defined with classes that inherit from `Synonym` @@ -231,6 +268,6 @@ class SongCollection(TypesenseCollection): ... ``` -To update the collection with any changes made to synonyms run `SongCollection().update_typesense_collection()` +To update the collection with any changes made to synonyms run `python manage.py updatecollections` diff --git a/django_typesense/__init__.py b/django_typesense/__init__.py index 485f44a..b3f4756 100644 --- a/django_typesense/__init__.py +++ b/django_typesense/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/django_typesense/changelist.py b/django_typesense/changelist.py index 2197c1e..b1d3c8c 100644 --- a/django_typesense/changelist.py +++ b/django_typesense/changelist.py @@ -15,6 +15,7 @@ from django.utils.dateparse import parse_datetime from django_typesense.fields import TYPESENSE_DATETIME_FIELDS +from django_typesense.utils import get_unix_timestamp # Changelist settings ALL_VAR = "all" @@ -80,8 +81,8 @@ def __init__( self.date_hierarchy = date_hierarchy self.search_fields = search_fields self.list_select_related = list_select_related - self.list_per_page = list_per_page - self.list_max_show_all = list_max_show_all + self.list_per_page = min(list_max_show_all, 250) # Typesense Max hits per page + self.list_max_show_all = min(list_max_show_all, 250) # Typesense Max hits per page self.model_admin = model_admin self.preserved_filters = model_admin.get_preserved_filters(request) self.sortable_by = sortable_by @@ -261,7 +262,7 @@ def get_search_filters(self, field_name: str, used_parameters: dict): if isinstance(field, tuple(TYPESENSE_DATETIME_FIELDS)): datetime_object = parse_datetime(value) - value = int(datetime.combine(datetime_object, datetime.min.time()).timestamp()) + value = get_unix_timestamp(datetime_object) if str(value).isdigit(): if lookup in ['gte', 'gt']: @@ -280,7 +281,14 @@ def get_search_filters(self, field_name: str, used_parameters: dict): if field.field_type == 'string': search_filters_dict[field_name] = f':{lookup_to_operator[lookup]}{value}' elif field.field_type == 'bool': - boolean_map = {'0': 'false', '1': 'true'} + value = value.lower() + boolean_map = { + '0': 'false', '1': 'true', + 'false': 'false', 'true': 'true', + 'no': 'false', 'yes': 'true', + 'n': 'false', 'y': 'true', + False: 'false', True: 'true', + } search_filters_dict[field_name] = boolean_map[value] else: search_filters_dict[field_name] = f'{lookup_to_operator[lookup]}{value}' @@ -312,13 +320,19 @@ def get_typesense_results(self, request): for filter_spec in self.filter_specs: if hasattr(filter_spec, 'filter_by'): - # ALL CUSTOM FILTERS + # all custom filters with filter_by defined filters_dict.update(filter_spec.filter_by) + continue if hasattr(filter_spec, 'field'): used_parameters = getattr(filter_spec, 'used_parameters') search_filters = self.get_search_filters(filter_spec.field.attname, used_parameters) filters_dict.update(search_filters) + else: + # custom filters where filter_by is not defined + used_parameters = getattr(filter_spec, 'used_parameters') + remaining_lookup_params.update(used_parameters) + for k, v in remaining_lookup_params.items(): try: @@ -342,7 +356,7 @@ def get_typesense_results(self, request): query, self.page_num, filter_by=filter_by, - sort_by=sort_by + sort_by=sort_by, ) # Set query string for clearing all filters. @@ -352,3 +366,18 @@ def get_typesense_results(self, request): ) return results + + def get_queryset(self, request): + # this is needed for admin actions that call cl.get_queryset + # To reduce the trips use max per_page + self.list_per_page = 250 + ids = [] + while True: + results = self.get_typesense_results(request) + if not results['hits']: + break + + ids.extend([result['document']['id'] for result in results['hits']]) + self.page_num += 1 + + return self.model.objects.filter(id__in=ids) diff --git a/django_typesense/collections.py b/django_typesense/collections.py index c0ac10d..dbfee43 100644 --- a/django_typesense/collections.py +++ b/django_typesense/collections.py @@ -75,12 +75,14 @@ def __init__( obj: Union[object, QuerySet, Iterable] = None, many: bool = False, data: list = None, + update_fields: list = None, ): assert ( self.query_by_fields ), "`query_by_fields` must be specified in the collection definition" assert not all([obj, data]), "`obj` and `data` cannot be provided together" + self.update_fields = update_fields self._meta = self._get_metadata() self.fields = self.get_fields() self._synonyms = [synonym().data for synonym in self.synonyms] @@ -186,7 +188,15 @@ def schema_fields(self) -> list: return [field.attrs for field in self.fields.values()] def _get_object_data(self, obj): - return {field.name: field.value(obj) for field in self.fields.values()} + if self.update_fields: + # we need the id for updates and a user can leave it out + update_fields = set(self.fields.keys()).intersection(set(self.update_fields)) + update_fields.add('id') + fields = [self.get_field(field_name) for field_name in update_fields] + else: + fields = self.fields.values() + + return {field.name: field.value(obj) for field in fields} @property def schema(self) -> dict: @@ -279,18 +289,34 @@ def delete(self): except ObjectNotFound: pass - def update(self): + def update(self, action_mode: str = "emplace"): if not self.data: return + if len(self.data) == 1: + return self._update_single_document(self.data[0]) + else: + return self._update_multiple_documents(action_mode) + + def _update_single_document(self, document): + document_id = document.pop('id') + + try: + return client.collections[self.schema_name].documents[document_id].update(document) + except ObjectNotFound: + self.create_typesense_collection() + document['id'] = document_id + return client.collections[self.schema_name].documents.upsert(document) + + def _update_multiple_documents(self, action_mode): try: return client.collections[self.schema_name].documents.import_( - self.data, {"action": "upsert"} + self.data, {"action": action_mode} ) except ObjectNotFound: self.create_typesense_collection() return client.collections[self.schema_name].documents.import_( - self.data, {"action": "upsert"} + self.data, {"action": action_mode} ) def create_or_update_synonyms(self): diff --git a/django_typesense/fields.py b/django_typesense/fields.py index 9f30dc3..a224e22 100644 --- a/django_typesense/fields.py +++ b/django_typesense/fields.py @@ -4,6 +4,7 @@ from typing import Optional from operator import attrgetter +from django_typesense.utils import get_unix_timestamp TYPESENSE_SCHEMA_ATTRS = [ 'name', '_field_type', 'sort', 'index', 'optional', 'facet', 'infix', 'locale' @@ -126,24 +127,27 @@ class TypesenseBooleanField(TypesenseField): _sort = True -class TypesenseDateTimeFieldBase(TypesenseBigIntegerField): +class TypesenseDateTimeFieldBase(TypesenseField): + _field_type = "int64" + _sort = True + def value(self, obj): _value = super().value(obj) + if _value is None: + return None + if isinstance(_value, int): return _value - # isinstance can take a union type but for backwards compatibility we call it multiple times - elif isinstance(_value, datetime): - _value = int(_value.timestamp()) - - elif isinstance(_value, date): - _value = int(datetime.combine(_value, datetime.min.time()).timestamp()) + _value = get_unix_timestamp(_value) - elif isinstance(_value, time): - _value = int(datetime.combine(datetime.today(), _value).timestamp()) - - return _value + try: + return int(_value) + except (TypeError, ValueError) as e: + raise e.__class__( + f"Field '{self.name}' expected a number but got {_value}.", + ) from e class TypesenseDateField(TypesenseDateTimeFieldBase): diff --git a/django_typesense/mixins.py b/django_typesense/mixins.py index 1da84a1..5f04463 100644 --- a/django_typesense/mixins.py +++ b/django_typesense/mixins.py @@ -17,7 +17,7 @@ def update(self, **kwargs): obj_ids = list(self.values_list('id', flat=True)) update_result = super().update(**kwargs) queryset = self.model.objects.filter(id__in=obj_ids) - collection = self.model.get_collection(queryset, many=True) + collection = self.model.get_collection(queryset, many=True, update_fields=kwargs.keys()) collection.update() return update_result diff --git a/django_typesense/utils.py b/django_typesense/utils.py index e9b5636..50819ff 100644 --- a/django_typesense/utils.py +++ b/django_typesense/utils.py @@ -1,18 +1,16 @@ import concurrent.futures import logging import os -from datetime import datetime +from datetime import datetime, date, time from django.db.models import QuerySet from django.core.paginator import Paginator -from django_typesense.collections import TypesenseCollection -from django_typesense.typesense_client import client logger = logging.getLogger(__name__) -def update_batch(documents_queryset: QuerySet, collection_class: TypesenseCollection, batch_no: int) -> None: +def update_batch(documents_queryset: QuerySet, collection_class, batch_no: int) -> None: """ Helper function that updates a batch of documents using the Typesense API. """ @@ -89,6 +87,7 @@ def bulk_delete_typesense_records(document_ids: list, collection_name: str) -> N None """ + from django_typesense.typesense_client import client try: client.collections[collection_name].documents.delete( {"filter_by": f"id:{document_ids}"} @@ -109,6 +108,7 @@ def typesense_search(collection_name, **kwargs): A list of the typesense results """ + from django_typesense.typesense_client import client if not collection_name: return @@ -125,10 +125,25 @@ def get_unix_timestamp(datetime_object): Get the unix timestamp from a datetime object with the time part set to midnight Args: - datetime_object: a python date/datetime object + datetime_object: a python date/datetime/time object Returns: An integer representing the timestamp """ - return int(datetime.combine(datetime_object, datetime.min.time()).timestamp()) + # isinstance can take a union type but for backwards compatibility we call it multiple times + if isinstance(datetime_object, datetime): + timestamp = int(datetime_object.timestamp()) + + elif isinstance(datetime_object, date): + timestamp = int(datetime.combine(datetime_object, datetime.min.time()).timestamp()) + + elif isinstance(datetime_object, time): + timestamp = int(datetime.combine(datetime.today(), datetime_object).timestamp()) + + else: + raise Exception( + f"Expected a date/datetime/time objects but got {datetime_object} of type {type(datetime_object)}" + ) + + return timestamp diff --git a/setup.py b/setup.py index 39c0ffd..183d182 100644 --- a/setup.py +++ b/setup.py @@ -16,4 +16,13 @@ long_description=open("README.md").read(), long_description_content_type="text/markdown", license_files=("LICENSE",), + classifiers=[ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Framework :: Django", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + ] )