In a C# boolean query Lucene search with multiple fields three of the fields are not included in the search (Sku, VariantSkus and Mpc), while the other fields are work just fine.
Using Luke, I can see that the values are stored in the index. When searching in Luke I get the correct results, using the query contained in the searcher (taken from the debugger in Visual Studio). Example: Using the following query: (taken directly from the query value while debugging in Visual Studio)
(+Mpc:B118^5) (+Sku:B118^5) (+Brand:B118) (+VariantSkus:B118^4) (+DisplayName:B118^3) (+DisplayName:B118*) (+DisplayName:B118~0.5) (+MisspelledNames:B118) (+Description:B118^0.4)
Doesn't work while running the code (totalHits is 0 on the searcher), but gives the expected result of matching the Mpc to the correct product in Luke.
I'm honestly quite confused as to why the same query does not work in the C# code. Any help or suggestions would be appreciated.
Creation of the index:
public static String CreateLuceneIndex(string basePath, HttpContext context)
{
var stopwatch = new Stopwatch();
/* get the absolute path to the directory where the indexes will be created (and if it doesn't exist, create it) */
var dirPath = context.Server.MapPath(basePath);
if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
var di = new DirectoryInfo(dirPath);
var directory = FSDirectory.Open(di);
stopwatch.Start();
/* Select the standard Lucene analyser */
var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29);
var count = 0;
var catalog = ProductCatalog.All().First();
/* Open the index writer using the selected analyser */
using (var writer = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED))
using(var mediaRepository = new ProductMediaRepository())
{
var urlService = ObjectFactory.Instance.Resolve<IUrlService>();
// Get all the visible products from uCommerce we wish to index
foreach (var product in Product.Find(p => p.DisplayOnSite && p.ParentProduct == null))
{
var url = urlService.GetUrl(catalog, product);
var doc = new Document();
doc.Add(new Field("id", product.Id.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES));
doc.Add(new Field("Url", url ?? String.Empty, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES));
doc.Add(new Field("Src", ImageService.GetProductMainImage(mediaRepository, product).Src ?? String.Empty
, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES));
doc.Add(new Field("Sku", product.Sku ?? String.Empty, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));
var varianSkus = String.Join(" ", product.Variants.Select(variant => variant.VariantSku));
doc.Add(new Field("VariantSkus", varianSkus, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));
doc.Add(new Field("DisplayName", product.DisplayName() ?? product.Name ?? String.Empty, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));
var brands = String.Join(" ", product.Variants.Select(variant => variant.GetPropertyValue<String>("Brand")).Where(w => !String.IsNullOrWhiteSpace(w)));
doc.Add(new Field("Brand", brands ?? String.Empty, Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
doc.Add(new Field("MisspelledNames", product.GetPropertyValue<String>("MisspelledNames") ?? String.Empty,
Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
doc.Add(new Field("Description", product.ShortDescription()?.StripHtml() ?? String.Empty, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));
doc.Add(new Field("Mpc", product.GetPropertyValue<String>("MPC") ?? String.Empty, Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.YES));
writer.AddDocument(doc);
count++;
}
writer.Optimize();
writer.Close();
}
stopwatch.Stop();
return $"Indexed {count} products in {stopwatch.Elapsed}.\n\n";
Searching:
public static ListItemsDtoModel ProductSearch(String searchTerm, String indexDirPath, Int32 maxResults = Int32.MaxValue)
{
searchTerm = searchTerm.Trim().ToLowerInvariant();
var searchWords = ParseSearchWords(searchTerm);
indexDirPath = HttpContext.Current.Server.MapPath(indexDirPath);
var di = new DirectoryInfo(indexDirPath);
using (var directory = FSDirectory.Open(di))
using (var searcher = new IndexSearcher(IndexReader.Open(directory, true)))
{
var query = new BooleanQuery();
query.Add(new BooleanClause(AddTermClauseGroup("Mpc", searchWords, 5), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddTermClauseGroup("Sku", searchWords, 5), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddTermClauseGroup("Brand", searchWords), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddTermClauseGroup("VariantSkus", searchWords, 4), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddTermClauseGroup("DisplayName", searchWords, 3), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddWildcardClauseGroup("DisplayName", searchWords), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddFuzzyTermClauseGroup("DisplayName", searchWords), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddTermClauseGroup("MisspelledNames", searchWords), BooleanClause.Occur.SHOULD));
query.Add(new BooleanClause(AddTermClauseGroup("Description", searchWords, 0.4f), BooleanClause.Occur.SHOULD));
var searchResults = searcher.Search(query, maxResults);
return AsListItemsDtoModel(searchResults.ScoreDocs.Select(sd =>
{
var document = searcher.Doc(sd.doc);
return new ImageLinkDtoModel
{
Url = document.Get("Url"),
Text = document.Get("DisplayName"),
Alt = document.Get("DisplayName"),
Src = document.Get("Src"),
};
}).ToList());
}
}
private static String[] ParseSearchWords(string searchTerm)
{
return searchTerm.Split(' ', '-')
.Where(w => !String.IsNullOrWhiteSpace(w))
.Select(QueryParser.Escape)
.ToArray();
}
private static BooleanQuery AddTermClauseGroup(String field, IEnumerable<String> searchTerms, float boost = 1f)
{
var boostStr = Math.Abs(boost-1f) > 0.001 ? "^" + boost.ToString(CultureInfo.InvariantCulture) : String.Empty;
return AddClauseGroup(searchTerms, word => new TermQuery(new Term(field, word + boostStr)));
}
private static BooleanQuery AddFuzzyTermClauseGroup(String field, IEnumerable<String> searchTerms)
{
return AddClauseGroup(searchTerms, word => new FuzzyQuery(new Term(field, word), 0.5f));
}
private static BooleanQuery AddWildcardClauseGroup(String field, IEnumerable<String> searchTerms)
{
return AddClauseGroup(searchTerms, word => new WildcardQuery(new Term(field, word + "*")));
}
private static BooleanQuery AddClauseGroup(IEnumerable<String> searchTerms, Func<String, Query> createSubClause)
{
var query = new BooleanQuery();
foreach (var searchTerm in searchTerms)
{
query.Add(new BooleanClause(createSubClause(searchTerm), BooleanClause.Occur.MUST));
}
return query;
}
