Refactor search code

readthedocs/core/static-src/core/js/doc-embed/search.js

@@ -32,6 +32,7 @@ function attach_elastic_search_query(data) {
                 var total_count = data.count || 0;
 
                 if (hit_list.length) {
+                    console.debug('Read the Docs search got a result. Showing results.')
                     for (var i = 0; i < hit_list.length; i += 1) {
                         var doc = hit_list[i];
                         var highlight = doc.highlight;
@@ -55,12 +56,18 @@ function attach_elastic_search_query(data) {
 
                         // Show highlighted texts
                         if (highlight.content) {
-                            var content_text = xss(highlight.content[0]);
-                            var contents = $('<div class="context">');
-
-                            contents.html(content_text);
-                            contents.find('em').addClass('highlighted');
-                            list_item.append(contents);
+                            for (index in highlight.content) {
+                                if (index < 3) {
+                                    // Show up to 3 results for search
+                                    var content = highlight.content[index];
+                                    var content_text = xss(content);
+                                    var contents = $('<div class="context">');
+
+                                    contents.html("..." + content_text + "...");
+                                    contents.find('em').addClass('highlighted');
+                                    list_item.append(contents);
+                                }
+                            }
                         }
 
                         Search.output.append(list_item);
@@ -71,10 +78,11 @@ function attach_elastic_search_query(data) {
                 if (!hit_list.length) {
                     // Fallback to Sphinx's indexes
                     Search.query_fallback(query);
+                    console.log('Read the Docs search failed. Falling back to Sphinx search.')
                 }
                 else {
                     Search.status.text(
-                        _('Search finished, found %s page(s) matching the search query.').replace('%s', total_count)
+                        _('Search finished, found %s page(s) matching the search query.').replace('%s', hit_list.length)
                     );
                 }
             })
@@ -97,6 +105,7 @@ function attach_elastic_search_query(data) {
             },
             complete: function (resp, status_code) {
                 if (status_code !== 'success' || resp.responseJSON.count === 0) {
+                    console.debug('Read the Docs search failed, skipping loading search content.')
                     return search_def.reject();
                 }
                 return search_def.resolve(resp.responseJSON);

readthedocs/projects/urls/public.py

@@ -8,6 +8,7 @@
 from readthedocs.constants import pattern_opts
 from readthedocs.projects.views import public
 from readthedocs.projects.views.public import ProjectDetailView, ProjectIndex
+from readthedocs.search import views as search_views
 
 
 urlpatterns = [
@@ -50,7 +51,7 @@
     ),
     url(
         r'^(?P<project_slug>{project_slug})/search/$'.format(**pattern_opts),
-        public.elastic_project_search,
+        search_views.elastic_project_search,
         name='elastic_project_search',
     ),
     url(

readthedocs/projects/views/public.py

@@ -25,9 +25,7 @@
 from readthedocs.builds.models import Version
 from readthedocs.builds.views import BuildTriggerMixin
 from readthedocs.projects.models import Project
-from readthedocs.search.documents import PageDocument
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
-from readthedocs.search.views import LOG_TEMPLATE
 
 from .base import ProjectOnboardMixin
 
@@ -240,48 +238,6 @@ def project_download_media(request, project_slug, type_, version_slug):
     return response
 
 
-def elastic_project_search(request, project_slug):
-    """Use elastic search to search in a project."""
-    queryset = Project.objects.protected(request.user)
-    project = get_object_or_404(queryset, slug=project_slug)
-    version_slug = request.GET.get('version', LATEST)
-    query = request.GET.get('q', None)
-    results = None
-    if query:
-        user = ''
-        if request.user.is_authenticated:
-            user = request.user
-        log.info(
-            LOG_TEMPLATE.format(
-                user=user,
-                project=project or '',
-                type='inproject',
-                version=version_slug or '',
-                language='',
-                msg=query or '',
-            ),
-        )
-
-    if query:
-        req = PageDocument.simple_search(query=query)
-        filtered_query = (
-            req.filter('term', project=project.slug)
-            .filter('term', version=version_slug)
-        )
-        paginated_query = filtered_query[:50]
-        results = paginated_query.execute()
-
-    return render(
-        request,
-        'search/elastic_project_search.html',
-        {
-            'project': project,
-            'query': query,
-            'results': results,
-        },
-    )
-
-
 def project_versions(request, project_slug):
     """
     Project version list view.

readthedocs/search/api.py

@@ -1,16 +1,53 @@
+import logging
+from pprint import pformat
+
 from rest_framework import generics
+from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
+from rest_framework.pagination import PageNumberPagination
 
 from readthedocs.search.documents import PageDocument
-from readthedocs.search.filters import SearchFilterBackend
-from readthedocs.search.pagination import SearchPagination
-from readthedocs.search.serializers import PageSearchSerializer
 from readthedocs.search.utils import get_project_list_or_404
 
+log = logging.getLogger(__name__)
+
+
+class SearchPagination(PageNumberPagination):
+    page_size = 25
+    page_size_query_param = 'page_size'
+    max_page_size = 100
+
+
+class PageSearchSerializer(serializers.Serializer):
+    project = serializers.CharField()
+    version = serializers.CharField()
+    title = serializers.CharField()
+    path = serializers.CharField()
+    link = serializers.SerializerMethodField()
+    highlight = serializers.SerializerMethodField()
+
+    def get_link(self, obj):
+        projects_url = self.context.get('projects_url')
+        if projects_url:
+            docs_url = projects_url[obj.project]
+            return docs_url + obj.path
+
+    def get_highlight(self, obj):
+        highlight = getattr(obj.meta, 'highlight', None)
+        if highlight:
+            if hasattr(highlight, 'content'):
+                for num, result in enumerate(highlight.content):
+                    # Change results to turn newlines in highlight into periods
+                    # https://github.com/rtfd/readthedocs.org/issues/5168
+                    new_text = result.replace('\n', '. ')
+                    highlight.content[num] = new_text
+            ret = highlight.to_dict()
+            log.debug('API Search highlight: %s', pformat(ret))
+            return ret
+
 
 class PageSearchAPIView(generics.ListAPIView):
     pagination_class = SearchPagination
-    filter_backends = [SearchFilterBackend]
     serializer_class = PageSearchSerializer
 
     def get_queryset(self):
@@ -24,7 +61,15 @@ def get_queryset(self):
         # Validate all the required params are there
         self.validate_query_params()
         query = self.request.query_params.get('q', '')
-        queryset = PageDocument.simple_search(query=query)
+        kwargs = {}
+        kwargs['projects_list'] = [p.slug for p in self.get_all_projects()]
+        kwargs['versions_list'] = self.request.query_params.get('version')
+        user = ''
+        if self.request.user.is_authenticated:
+            user = self.request.user
+        queryset = PageDocument.faceted_search(
+            query=query, user=user, **kwargs
+        )
         return queryset
 
     def validate_query_params(self):
@@ -43,13 +88,15 @@ def get_serializer_context(self):
         context['projects_url'] = self.get_all_projects_url()
         return context
 
-    def get_all_projects_url(self):
-        version_slug = self.request.query_params.get('version')
+    def get_all_projects(self):
         project_slug = self.request.query_params.get('project')
         all_projects = get_project_list_or_404(project_slug=project_slug, user=self.request.user)
-        projects_url = {}
+        return all_projects
 
+    def get_all_projects_url(self):
+        all_projects = self.get_all_projects()
+        version_slug = self.request.query_params.get('version')
+        projects_url = {}
         for project in all_projects:
             projects_url[project.slug] = project.get_docs_url(version_slug=version_slug)
-
         return projects_url

readthedocs/search/apps.py

@@ -1,10 +1,8 @@
-"""Project app config"""
-
 from django.apps import AppConfig
 
 
 class SearchConfig(AppConfig):
     name = 'readthedocs.search'
 
     def ready(self):
-        from .signals import index_html_file, remove_html_file
+        import readthedocs.search.signals  # noqa