Full text Search with Multiple index in Elastic Search using NEST C#

Question

I'm trying to search Multiple indexes Elasticsearch with NEST Client, I just follow the below link
[stackover post ]How to search inside multiple indices using Nest ElasticSearch? the only difference was my indexes are already existed but nothing returns

Sample code :

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Elasticsearch.Net;
using Nest;

namespace ElasticSearchDemo
{


    public class ExceptionData
    {
        public bool HasException { get; set; }
        public string ExceptionMessage { get; set; }
    }
    public class ElasticSearchResponse : ExceptionData
    {
        public ISearchResponse<dynamic> elasticSearchResponse { get; set; }
    }

    public class ComponentTypES
    {
        public string ComponentID { get; set; }
        public string Componentname { get; set; }
        public string Summary { get; set; }
    }

    public class ProjectTypES
    {
        public string ProjectID { get; set; }
        public string Projectname { get; set; }
        public string Summary { get; set; }
        public string Description { get; set; }
    }
    class Program
    {
        static void Main(string[] args)
        {
            // calling the function
            var response = GetAllSearchResults("test", 0, 10);



        }

        public static ElasticClient GetElasticSearchCommonSearch()
        {
            ElasticClient elasticClient = null;
            try
            {
                const string strElasticSearchURL = "http://localhost:9200/";
                const string componentIndex = "componenttypeindex";
                const string projectIndex = "projecttypeindex";

                if (!string.IsNullOrEmpty(strElasticSearchURL))
                {
                    ConnectionSettings connectionSettings = new ConnectionSettings(new Uri(strElasticSearchURL))
                        .DefaultIndex(componentIndex)
                        .DefaultMappingFor<ComponentTypES>(i => i.IndexName(componentIndex).TypeName("Componenttype"))
                        .DefaultMappingFor<ProjectTypES>(j => j.IndexName(projectIndex).TypeName("Projecttype"))

                        .DisableDirectStreaming()
                        .PrettyJson()
                                .OnRequestCompleted(callDetails =>
                                {
                                    if (callDetails.RequestBodyInBytes != null)
                                    {
                                        Console.WriteLine(
                                            $"{callDetails.HttpMethod} {callDetails.Uri} \n" +
                                            $"{Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
                                    }
                                    else
                                    {
                                        Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
                                    }

                                    Console.WriteLine();

                                    if (callDetails.ResponseBodyInBytes != null)
                                    {
                                        Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
                                                 $"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}\n" +
                                                 $"{new string('-', 30)}\n");
                                    }
                                    else
                                    {
                                        Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
                                                 $"{new string('-', 30)}\n");
                                    }
                                }
                        );

                    elasticClient = new ElasticClient(connectionSettings);
                }

            }
            catch (Exception ex)
            {
                throw new Exception(ex.Message + "  ConnectionObject for : Common Search");
            }

            return elasticClient;
        }

        public static ElasticSearchResponse GetAllSearchResults(string query = "test", int
                                              page = 1, int pagesize = 10)
        {
            ElasticSearchResponse combinedResponse = new   ElasticSearchResponse();

            try
            {
                ElasticClient elasticClient =  GetElasticSearchCommonSearch();

                var clusterHealth = elasticClient.ClusterHealth();
                if (clusterHealth.IsValid && string.Compare(clusterHealth.Status.ToString(), "red", true) != 0 && clusterHealth.ServerError == null)
                {
                    string Componentindex = "componenttypeindex";
                    string Projectindex =  "projecttypeindex";

                    var indices = Indices.Index(typeof(ComponentTypES)).And(typeof(ProjectTypES));

                    //elasticClient.Refresh(indices);

                    //TODO : Development time coding 

                    if (null != (indices))
                    {
                        var indexExists = elasticClient.IndexExists(Indices.Index(Componentindex));
                        var projectExists = elasticClient.IndexExists(Indices.Index(Projectindex));

                        if (indexExists.Exists && indexExists.IsValid && projectExists.Exists && projectExists.IsValid)
                        {


                            //full text example 1

                            combinedResponse.elasticSearchResponse = elasticClient.Search<object>(s => s
                             .Index(indices)
                             .Type(Types.Type(typeof(ComponentTypES), typeof(ProjectTypES)))
                             .Query(q => (q
                             .MultiMatch(m => m
                              .Fields(f => f
                                      .Field(Infer.Field<ComponentTypES>(ff => ff.Componentname))
                                      .Field(Infer.Field<ComponentTypES>(ff => ff.Summary, 1.1))
                                        )
                              .Operator(Operator.Or)
                              .Query(query)
                                         ) && +q
                             .Term("_index", Componentindex)) || (q
                             .MultiMatch(m => m
                             .Fields(f => f
                                         .Field(Infer.Field<ProjectTypES>(ff => ff.Projectname))
                                         .Field(Infer.Field<ProjectTypES>(ff => ff.Summary, 0.3))
                              )
                              .Operator(Operator.Or)
                              .Query(query)
                               ) && +q
                                 .Term("_index", Projectindex))
                              ).From(page - 1)
                              .Size(pagesize)

                               );


                            //free text example 2
                            combinedResponse.elasticSearchResponse = elasticClient.Search<object>(s => s
                                                             .Index(indices)
                                                             .Type(Types.Type(typeof(ComponentTypES), typeof(ProjectTypES)))
                                                             .Query(q => (q
                                                                 .MatchPhrase(m => m
                                                                         .Field(Infer.Field<ComponentTypES>(ff => ff.Componentname))
                                                                         .Query(query)
                                                                 ) && +q
                                                                 .Term("_index", Componentindex)) || (q
                                                                 .MatchPhrase(m => m
                                                                     .Field(Infer.Field<ProjectTypES>(ff => ff.Projectname))
                                                                     .Query(query)
                                                                     )
                                                                 ) && +q
                                                                 .Term("_index", Projectindex)
                                                             ).From(page - 1)
                                                             .Size(pagesize)
                                                        );



                        }
                        else
                        {
                            combinedResponse.HasException = true;
                            combinedResponse.ExceptionMessage = "Index Not Found";
                        }
                    }
                    else
                    {
                        combinedResponse.HasException = true;
                        combinedResponse.ExceptionMessage = "Index Not Found In Config File";
                    }
                }
                else
                {
                    combinedResponse.HasException = true;
                    combinedResponse.ExceptionMessage = "Error on connecting with ElasticSearch";
                }
            }
            catch (Exception ex)
            {
                combinedResponse.HasException = true;
                combinedResponse.ExceptionMessage = ex.Message;
                return combinedResponse;
            }

            return combinedResponse;
        }


    }
}

Elastic table schema:

PUT componenttypeindex
{
  "mappings": {
    "Componenttype":{
      "properties":{
        "ComponentID":{"type":"text"},
        "Componentname":{"type":"text"},
        "Summary":{"type":"text"}
           }
        }
    }
}

PUT projecttypeindex
{
  "mappings": {
    "Projecttype":{
      "properties":{
        "ProjectID":{"type":"text"},
        "Projectname":{"type":"text"},
        "Summary":{"type":"text"},
         "Description":{"type":"text"}
                }
            }
         }
}

it should return query matched items, but nothing returns sorry for my ugly code formatting I tried but the new editor won't change anything

UPDATE : i've updated the Index values in the query as suggested by @RussCam but still no expected results , and also when expands the response objects and ran the URI parameter in directly in Browser it has all the results something weird not sure why this not shown in response count

Valid NEST response built from a successful low level call on POST: /componenttypeindex%2Cprojecttypeindex/Componenttype%2CProjecttype/_search?typed_keys=true

Audit trail of this API call:

[1] Healthy Response: Node: http://localhost:9200/ Took: 00:00:00.0620000
Request:

URI = "http://localhost:9200/componenttypeindex%2Cprojecttypeindex/Componenttype%2CProjecttype/_search?typed_keys=true"

My POCO Classes:

public class ComponentTypES
{
    public string ComponentID { get; set; }
    public string Componentname { get; set; }
    public string Summary { get; set; }

}  

public class ProjectTypES
{

    public string ProjectID { get; set; }
    public string Projectname { get; set; }
    public string Summary { get; set; }
    public string Description { get; set; } 
}

sample Data :

PUT componenttypeindex/Componenttype/5342e739-1635-4021-baf2-55e25b95b8ec
{
    "ComponentID":"5342e739-1635-4021-baf2-55e25b95b8ec",
    "Componentname":"TestComponent1",
    "Summary":"this is summary of test component1"
}

PUT componenttypeindex/Componenttype/90781386-8065-11e9-bc42-526af7764f64
{    
    "ComponentID":"90781386-8065-11e9-bc42-526af7764f64",
    "Componentname":"TestComponent2",
    "Summary":"this is summary of test component3"  
}
PUT componenttypeindex/Componenttype/19871386-8065-11e9-bc42-526af7764f64
{
    "ComponentID":"19871386-8065-11e9-bc42-526af7764f64",
    "Componentname":"some xyz component test",
    "Summary":"this is summary test of test xyz"
}


PUT projecttypeindex/Projecttype/5342e739-2019-4021-baf2-55e25b95b8ec
{
        "ProjectID":"5342e739-2019-4021-baf2-55e25b95b8ec",
        "Projectname":"Test Project1",
        "Summary":"summary of Test Project1",
        "Description":"Description of TestProject1"
}

PUT projecttypeindex/Projecttype/5342f739-2019-4021-baf2-55e25b95b8ba
{
        "ProjectID":"5342f739-2019-4021-baf2-55e25b95b8ba",
        "Projectname":"Test Project2",
        "Summary":"summary of Test Project2",
        "Description":"Description of TestProject1"
}

PUT projecttypeindex/Projecttype/6342f739-2020-4021-baf2-55e25b95b8ac
{
        "ProjectID":"6342f739-2020-4021-baf2-55e25b95b8ac",
        "Projectname":"some PQRS project",
        "Summary":"summary of PQRS Project",
        "Description":"Description of PQORS Project1"
}

What version of Elasticsearch are you using? What version of NEST are you using? — Russ Cam
@RussCam ElasticSearch.NET version 6.4.0 , NEST version 6.4.0 , please let me know if you need any other information — Chandru velan
A complete example would really help as the question only has partial information in it. Where you use .Term("_index", "Componenttype"), it looks like you could use componentIndex and similarly, where you use .Term("_index", "Projecttype"), it looks like you could use projectIndex — Russ Cam
@RussCam Thanks for your comments will give try with the change you are suggested and i'll add additonal information like POCO objects as well — Chandru velan
@RussCam i've added the requested additional information please let me if you need any other details , i've ran the Response object URI parameter which returns the results but the hits count always zero , please suggest — Chandru velan

Russ Cam Russ Cam · Accepted Answer · 2019-06-13T01:04:30

There's a lot of superfluous information in your example that makes it tricky to work with, which raises the barrier of effort required by someone wishing to help. Could I suggest that you reduce an example down to the smallest, succinct but complete example that demonstrates the problem you are facing in future; it'll really help in getting to the crux of the issue quicker!

I think the fundamental issue is that the casing of properties within the fields in the index mappings, and the casing of properties that NEST will send by default are different, so the nested must clauses within the should clause of the bool query that is generated by NEST will never be matched by any documents because of the field casing difference.

NEST by default camel cases property names, but the fields in the index mappings and documents in your example are all pascal cased, so the field names generated by NEST will not match those within the mapping. You can easily change NEST's field casing behaviour by using the DefaultFieldNameInferrer(Func<string, string>) method on ConnectionSettings. A delegate that simply returns the string value passed will leave field names as they are on the POCOs.

Here's a complete but succinct, working example

private static void Main()
{
    const string componentIndex = "componenttypeindex";
    const string projectIndex = "projecttypeindex";

    var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));

    var settings = new ConnectionSettings(pool)
        .DefaultIndex(componentIndex)
        .DefaultMappingFor<ComponentTypES>(i => i.IndexName(componentIndex).TypeName("Componenttype").IdProperty(f => f.ComponentID))
        .DefaultMappingFor<ProjectTypES>(j => j.IndexName(projectIndex).TypeName("Projecttype").IdProperty(f => f.ProjectID))
        .DefaultFieldNameInferrer(f => f)
        .DefaultTypeName("_doc")
        .DisableDirectStreaming()
        .PrettyJson()
        .OnRequestCompleted(callDetails =>
        {
            if (callDetails.RequestBodyInBytes != null)
            {
                Console.WriteLine(
                    $"{callDetails.HttpMethod} {callDetails.Uri} \n" +
                    $"{Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
            }
            else
            {
                Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
            }

            Console.WriteLine();

            if (callDetails.ResponseBodyInBytes != null)
            {
                Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
                         $"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}\n" +
                         $"{new string('-', 30)}\n");
            }
            else
            {
                Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
                         $"{new string('-', 30)}\n");
            }
        });

    var client = new ElasticClient(settings);

    foreach (var index in new[] { componentIndex, projectIndex }) 
    {
        if (client.IndexExists(index).Exists)
            client.DeleteIndex(index);

        client.CreateIndex(index, c => c
            .Mappings(m => {
                if (index == projectIndex)
                    return m.Map<ProjectTypES>(mm => mm.AutoMap());
                else
                    return m.Map<ComponentTypES>(mm => mm.AutoMap());
            })
        );
    }

    client.Bulk(b => b
        .IndexMany(new [] {
            new ComponentTypES 
            {
                ComponentID = "5342e739-1635-4021-baf2-55e25b95b8ec",
                Componentname = "TestComponent1",
                Summary = "this is summary of test component1"
            },
            new ComponentTypES
            {
                ComponentID = "90781386-8065-11e9-bc42-526af7764f64",
                Componentname = "TestComponent2",
                Summary = "this is summary of test component3"
            },
            new ComponentTypES
            {
                ComponentID = "19871386-8065-11e9-bc42-526af7764f64",
                Componentname = "some xyz component test",
                Summary = "this is summary test of test xyz"
            },
        })
        .IndexMany(new [] {
            new ProjectTypES
            {
                ProjectID = "5342e739-2019-4021-baf2-55e25b95b8ec",
                Projectname = "Test Project1",
                Summary = "summary of Test Project1",
                Description = "Description of TestProject1"
            },
            new ProjectTypES
            {
                ProjectID = "5342f739-2019-4021-baf2-55e25b95b8ba",
                Projectname = "Test Project2",
                Summary = "summary of Test Project2",
                Description = "Description of TestProject1"
            },
            new ProjectTypES
            {
                ProjectID = "6342f739-2020-4021-baf2-55e25b95b8ac",
                Projectname = "some PQRS project",
                Summary = "summary of PQRS Project",
                Description = "Description of PQORS Project1"
            },
        })
        .Refresh(Refresh.WaitFor)
    );

    var query = "test";

    var response = client.Search<object>(s => s
        .Index(Indices.Index(typeof(ComponentTypES)).And(typeof(ProjectTypES)))
        .Type(Types.Type(typeof(ComponentTypES), typeof(ProjectTypES)))
        .Query(q => 
            (q
                .MultiMatch(m => m
                    .Fields(f => f
                        .Field(Infer.Field<ComponentTypES>(ff => ff.Componentname))
                        .Field(Infer.Field<ComponentTypES>(ff => ff.Summary, 1.1))
                    )
                    .Operator(Operator.Or)
                    .Query(query)
                ) && +q
                .Term("_index", componentIndex)
            ) || 
            (q
                .MultiMatch(m => m
                    .Fields(f => f
                        .Field(Infer.Field<ProjectTypES>(ff => ff.Projectname))
                        .Field(Infer.Field<ProjectTypES>(ff => ff.Summary, 0.3))
                    )
                    .Operator(Operator.Or)
                    .Query(query)
                ) && +q
                .Term("_index", projectIndex)
            )
        )
    );
}

public class ComponentTypES
{
    public string ComponentID { get; set; }
    public string Componentname { get; set; }
    public string Summary { get; set; }

}

public class ProjectTypES
{

    public string ProjectID { get; set; }
    public string Projectname { get; set; }
    public string Summary { get; set; }
    public string Description { get; set; }
}

The resulting JSON query for the search is

POST http://localhost:9200/componenttypeindex%2Cprojecttypeindex/Componenttype%2CProjecttype/_search?pretty=true&typed_keys=true 
{
  "query": {
    "bool": {
      "should": [
        {
          "bool": {
            "filter": [
              {
                "term": {
                  "_index": {
                    "value": "componenttypeindex"
                  }
                }
              }
            ],
            "must": [
              {
                "multi_match": {
                  "fields": [
                    "Componentname",
                    "Summary^1.1"
                  ],
                  "operator": "or",
                  "query": "test"
                }
              }
            ]
          }
        },
        {
          "bool": {
            "filter": [
              {
                "term": {
                  "_index": {
                    "value": "projecttypeindex"
                  }
                }
              }
            ],
            "must": [
              {
                "multi_match": {
                  "fields": [
                    "Projectname",
                    "Summary^0.3"
                  ],
                  "operator": "or",
                  "query": "test"
                }
              }
            ]
          }
        }
      ]
    }
  }
}

which returns 5 results

{
  "took" : 53,
  "timed_out" : false,
  "_shards" : {
    "total" : 10,
    "successful" : 10,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 5,
    "max_score" : 0.7549128,
    "hits" : [
      {
        "_index" : "projecttypeindex",
        "_type" : "Projecttype",
        "_id" : "5342e739-2019-4021-baf2-55e25b95b8ec",
        "_score" : 0.7549128,
        "_source" : {
          "ProjectID" : "5342e739-2019-4021-baf2-55e25b95b8ec",
          "Projectname" : "Test Project1",
          "Summary" : "summary of Test Project1",
          "Description" : "Description of TestProject1"
        }
      },
      {
        "_index" : "componenttypeindex",
        "_type" : "Componenttype",
        "_id" : "19871386-8065-11e9-bc42-526af7764f64",
        "_score" : 0.5565415,
        "_source" : {
          "ComponentID" : "19871386-8065-11e9-bc42-526af7764f64",
          "Componentname" : "some xyz component test",
          "Summary" : "this is summary test of test xyz"
        }
      },
      {
        "_index" : "componenttypeindex",
        "_type" : "Componenttype",
        "_id" : "5342e739-1635-4021-baf2-55e25b95b8ec",
        "_score" : 0.3164503,
        "_source" : {
          "ComponentID" : "5342e739-1635-4021-baf2-55e25b95b8ec",
          "Componentname" : "TestComponent1",
          "Summary" : "this is summary of test component1"
        }
      },
      {
        "_index" : "projecttypeindex",
        "_type" : "Projecttype",
        "_id" : "5342f739-2019-4021-baf2-55e25b95b8ba",
        "_score" : 0.2876821,
        "_source" : {
          "ProjectID" : "5342f739-2019-4021-baf2-55e25b95b8ba",
          "Projectname" : "Test Project2",
          "Summary" : "summary of Test Project2",
          "Description" : "Description of TestProject1"
        }
      },
      {
        "_index" : "componenttypeindex",
        "_type" : "Componenttype",
        "_id" : "90781386-8065-11e9-bc42-526af7764f64",
        "_score" : 0.20706992,
        "_source" : {
          "ComponentID" : "90781386-8065-11e9-bc42-526af7764f64",
          "Componentname" : "TestComponent2",
          "Summary" : "this is summary of test component3"
        }
      }
    ]
  }
}

Full text Search with Multiple index in Elastic Search using NEST C#

Audit trail of this API call:

Request:

1 Answers