Django elasticsearch dsl term and phrase search is not working

Question

I used two packages (i.e. django-elasticsearch-dsl==7.1.4 and django-elasticsearch-dsl-drf==0.20.8) to add a search engine to my Django project. The model which I indexed in elastic is:

class Article(models.Model):
    created_time = models.DateTimeField(_('created time'), auto_now_add=True)
    updated_time = models.DateTimeField(_('updated time'), auto_now=True)
    profile = models.ForeignKey('accounts.UserProfile', verbose_name=_('profile'), on_delete=models.PROTECT)
    approved_user = models.ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('approved user'), blank=True, null=True, editable=False, on_delete=models.CASCADE, related_name='article_approved_users')
    approved_time = models.DateTimeField(_('approved time'), blank=True, null=True, db_index=True, editable=False)
    title = models.CharField(_('title'), max_length=50)
    image = models.ImageField(_('image'), blank=True, upload_to=article_directory_path)
    slug = models.SlugField(_('slug'), max_length=50, unique=True)
    content = models.TextField(_('content'))
    summary = models.TextField(_('summary'))
    views_count = models.PositiveIntegerField(verbose_name=_('views count'), default=int, editable=False)
    is_free = models.BooleanField(_('is free'), default=True)
    is_enable = models.BooleanField(_('is enable'), default=True)

    tags = TaggableManager(verbose_name=_('tags'), related_name='articles')
    categories = models.ManyToManyField('Category', verbose_name=_('categories'), related_name='articles')
    namads = models.ManyToManyField('namads.Namad', verbose_name=_('namads'), related_name='articles', blank=True)

I used the following document for indexing my Article model:

html_strip = analyzer(
    'html_strip',
    tokenizer="whitespace",
    filter=["lowercase", "stop", "snowball"],
    char_filter=["html_strip"]
)


@registry.register_document
class ArticleDocument(Document):
    title = fields.TextField(
        analyzer=html_strip,
        fields={
            'raw': fields.TextField(analyzer='keyword'),
            'suggest': fields.CompletionField(),
        }
    )

    tags = fields.ObjectField(
        properties={
            "name": fields.TextField(
                analyzer=html_strip,
                fields={
                    'raw': fields.TextField(analyzer='keyword'),
                    'suggest': fields.CompletionField(),
                }
            )
        }
    )
    categories = fields.ObjectField(
        properties={
            'id': fields.IntegerField(),
            'title': fields.TextField(
                analyzer=html_strip,
                fields={
                    'raw': fields.TextField(analyzer='keyword'),
                    'suggest': fields.CompletionField(),
                }
            )
        }
    )
    namads = fields.ObjectField(
        properties={
            "id": fields.IntegerField(),
            "name": fields.TextField(
                analyzer=html_strip,
                fields={
                    'raw': fields.TextField(analyzer='keyword'),
                    'suggest': fields.CompletionField(),
                }
            ),
            "group_name": fields.TextField(
                analyzer=html_strip,
                fields={
                    'raw': fields.TextField(analyzer='keyword'),
                    'suggest': fields.CompletionField(),
                }
            )
        }
    )

    class Index:
        name = settings.ARTICLE_INDEX_NAME
        settings = {
            "number_of_shards": 1,
            "number_of_replicas": 0
        }

    def get_queryset(self):
        return super(ArticleDocument, self).get_queryset().filter(
            approved_user__isnull=False,
            is_enable=True
        ).prefetch_related(
            'tags',
            'categories',
            'namads'
        )

    class Django:
        model = Article
        fields = ['id', 'summary']

And finally used the following viewset to search on it's result (based on this document.)

class ArticleSearchViewSet(DocumentViewSet):
    """

        list:
            Search on all articles, ordered by most recently added.

            query parameters
            -  Search fields: 'title', 'summary', 'tags.name', 'categories.title', 'namads.name',
                'namads.group_name' . Ex: ?search=some random name.
        retrieve:
            Return a specific article details.

    """
    serializer_class = ArticleDocumentSerializer
    document = ArticleDocument
    pagination_class = PageNumberPagination
    lookup_field = 'id'
    filter_backends = [
        FilteringFilterBackend,
        IdsFilterBackend,
        OrderingFilterBackend,
        DefaultOrderingFilterBackend,
        CompoundSearchFilterBackend,
        SuggesterFilterBackend,
    ]
    search_fields = (
        'title',
        'summary',
        'tags.name',
        'categories.title',
        'namads.name',
        'namads.group_name'
    )
    filter_fields = {
        'id': {
            'field': 'id',
            # Note, that we limit the lookups of id field in this example,
            # to `range`, `in`, `gt`, `gte`, `lt` and `lte` filters.
            'lookups': [
                LOOKUP_FILTER_RANGE,
                LOOKUP_QUERY_IN,
                LOOKUP_QUERY_GT,
                LOOKUP_QUERY_GTE,
                LOOKUP_QUERY_LT,
                LOOKUP_QUERY_LTE,
            ],
        },
        'namads': {
            'field': 'namads',
            # Note, that we limit the lookups of `pages` field in this
            # example, to `range`, `gt`, `gte`, `lt` and `lte` filters.
            'lookups': [
                LOOKUP_FILTER_RANGE,
                LOOKUP_QUERY_GT,
                LOOKUP_QUERY_GTE,
                LOOKUP_QUERY_LT,
                LOOKUP_QUERY_LTE,
            ],
        },
        'title': "title.raw",
        'summary': 'summary',
        'categories': {
            'field': 'categories',
            # Note, that we limit the lookups of `pages` field in this
            # example, to `range`, `gt`, `gte`, `lt` and `lte` filters.
            'lookups': [
                LOOKUP_FILTER_RANGE,
                LOOKUP_QUERY_GT,
                LOOKUP_QUERY_GTE,
                LOOKUP_QUERY_LT,
                LOOKUP_QUERY_LTE,
            ],
        },

        'tags': {
            'field': 'tags',
            # Note, that we limit the lookups of `tags` field in
            # this example, to `terms, `prefix`, `wildcard`, `in` and
            # `exclude` filters.
            'lookups': [
                LOOKUP_FILTER_TERMS,
                LOOKUP_FILTER_PREFIX,
                LOOKUP_FILTER_WILDCARD,
                LOOKUP_QUERY_IN,
                LOOKUP_QUERY_EXCLUDE,
            ],
        },
    }
    # Suggester fields
    suggester_fields = {
        'title_suggest': {
            'field': 'title.suggest',
            'suggesters': [
                SUGGESTER_TERM,
                SUGGESTER_COMPLETION,
                SUGGESTER_PHRASE,

            ],
            'default_suggester': SUGGESTER_COMPLETION,
            'options': {
                'size': 10,  # Number of suggestions to retrieve.
                'skip_duplicates': True,  # Whether duplicate suggestions should be filtered out.
            },
        },
        'tags_suggest': {
            'field': 'tags.name.suggest',
            'suggesters': [
                SUGGESTER_COMPLETION,
            ],
            # 'options': {
            #     'size': 20,  # Override default number of suggestions
            # },
        },
        'categories_suggest': {
            'field': 'categories.title.suggest',
            'suggesters': [
                SUGGESTER_TERM,
                SUGGESTER_COMPLETION,
                SUGGESTER_PHRASE,
            ],
        },
        'namads_name_suggest': {
            'field': 'namads.name.suggest',
            'suggesters': [
                SUGGESTER_COMPLETION,
            ],
        },
        'namad_group_name_suggest': {
            'field': 'namads.group_name.suggest',
            'suggesters': [
                SUGGESTER_COMPLETION,
            ],
        },

    }
    ordering_fields = {
        'id': 'id',
        'title': 'title',
        'summary': 'summary',
    }
    # Specify default ordering
    ordering = ('-id', )

And my document serializer is:

class ArticleDocumentSerializer(DocumentSerializer):
    class Meta:
        document = ArticleDocument
        fields = ['id', 'title', 'summary', 'namads', 'categories', 'tags']

Every thing works fine except when I'm using term and phrase search with following query parameters:

?title_suggest__phrase=fi

?title_suggest__term=fi

In the url localhost/api/v1/blog/articles-search/suggest/ but in both cases the results are the same like the following:

{
    "title_suggest__term": [
        {
            "text": "fi",
            "offset": 0,
            "length": 2,
            "options": []
        }
    ]
}.

And also I'm pretty sure that I have First article in my index and when I use completion suggester every thing works fine (i.e. ?title_suggest__completion=fi) and it returns results. Am I missing something? I want to add term and phrase search to my project. How can I fix this problem (Term and phrase results are 0)?

Roham Roham · Accepted Answer · 2020-09-22T12:45:38

The problem was in my suggester configurations. First of all for term and phrase suggest we do not need completion fields (i.e. 'suggest': fields.CompletionField()) and we just need to declare our field in our Index, something like:

title = fields.TextField(
        fields={
            'raw': fields.TextField(analyzer=html_strip)
        }
    ) # which goes in documents.py

and just add the following to any fields to enable term and phrase suggest:

suggester_fields = {
        'title_suggest': {
            'field': 'title',
            'suggesters': [
                SUGGESTER_TERM,
                SUGGESTER_PHRASE,

            ],
        },
    } # Which goes in views.py and search view suggester_fields

and to see the related suggest result we should send query parameters like ?title_suggest__term=something or ?title_suggest__phrase=something. Finally if we need to add completion to suggester fields too we should add it with some other key like:

suggester_fields = {
        'title_suggest': {
            'field': 'title',
            'suggesters': [
                SUGGESTER_TERM,
                SUGGESTER_PHRASE,

            ],
        },
        'title': {
            'field': 'title',
            'suggesters': [
                SUGGESTER_COMPLETION,
            ],
        },
    }

and now with this configs we have three type of suggestions on title field (i.e. term, phrase and completion). So if we want the whole result based on these three suggesters we should use three query parameters like:

localhost/api/v1/blog/articles-search/suggest/?title_suggest__term=something&title_suggest__phrase=something&title__completion=something

And do not forget to change the field config in your Index(In case we need completion suggester):

title = fields.TextField(
        fields={
            'raw': fields.TextField(analyzer=html_strip),
            'suggest': fields.CompletionField(),
        }
    ) # Which goes in documents.py

Django elasticsearch dsl term and phrase search is not working

1 Answers