Skip to content

Commit

Permalink
Merge pull request geonetwork#395 from jesseeichar/FacetSearchSuggest…
Browse files Browse the repository at this point in the history 
…ions

Reimplement getSearchSuggestion Fields in LuceneSearch using facets
  • Loading branch information
Jesse Eichar committed Feb 22, 2014
2 parents 329456f + 54685d6 commit 731cd34
Show file tree
Hide file tree
Showing 11 changed files with 309 additions and 166 deletions.
2 changes: 2 additions & 0 deletions core/src/main/java/org/fao/geonet/constants/Geonet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ public static final class SearchResult {
public static final String BUILD_SUMMARY = "buildSummary";
public static final String SUMMARY_ONLY = "summaryOnly";
public static final String REQUESTED_LANGUAGE = "requestedLanguage";
public static final String SUMMARY_ITEMS = "summaryItems";

/**
* TODO javadoc.
Expand All @@ -425,6 +426,7 @@ public static final class ResultType {
* of the current search.
*/
public static final String RESULTS_WITH_SUMMARY = "results_with_summary";
public static final String SUGGESTIONS = "suggestions";
}

/**
Expand Down
17 changes: 16 additions & 1 deletion core/src/main/java/org/fao/geonet/kernel/search/LuceneConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,17 @@ public FacetConfig(Element summaryElement) {
sortOrder = Facet.SortOrder.ASCENDING;
}
}

public FacetConfig(FacetConfig config) {
this.name = config.name;
this.plural = config.plural;
this.indexKey = config.indexKey;
this.translator = config.translator;
this.max = config.max;
this.sortBy = sortBy;
this.sortOrder = sortOrder;
}

public String toString() {
StringBuffer sb = new StringBuffer("Field: ");
sb.append(indexKey);
Expand Down Expand Up @@ -212,14 +223,18 @@ public Facet.SortOrder getSortOrder() {
public int getMax() {
return max;
}
public void setMax(int max) {
this.max = max;
}
public Translator getTranslator(ServiceContext context, String langCode) {
try {
return Translator.createTranslator(translator, context, langCode);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

}

/**
* List of taxonomy by taxonomy types (hits, hits_with_summary
Expand Down
217 changes: 106 additions & 111 deletions core/src/main/java/org/fao/geonet/kernel/search/LuceneSearcher.java
View file

Large diffs are not rendered by default.

34 changes: 12 additions & 22 deletions core/src/main/java/org/fao/geonet/kernel/search/SearchManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,22 +94,8 @@
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
import java.util.Vector;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

Expand Down Expand Up @@ -671,9 +657,9 @@ public void index(String schemaDir, Element metadata, String id, List<Element> m
indexGeometry(schemaDir, metadata, id, moreFields);

// Update Lucene index
List<Pair<String, Pair<Document, List<CategoryPath>>>> docs = buildIndexDocument(schemaDir, metadata, id, moreFields, metadataType, title, false);
List<Pair<String, Pair<Document, Collection<CategoryPath>>>> docs = buildIndexDocument(schemaDir, metadata, id, moreFields, metadataType, title, false);
_tracker.deleteDocuments(new Term("_id", id));
for( Pair<String, Pair<Document, List<CategoryPath>>> document : docs ) {
for( Pair<String, Pair<Document, Collection<CategoryPath>>> document : docs ) {
_tracker.addDocument(document.one(), document.two().one(), document.two().two());
if(Log.isDebugEnabled(Geonet.INDEX_ENGINE)) {
Log.debug(Geonet.INDEX_ENGINE, "adding document in locale " + document.one());
Expand Down Expand Up @@ -731,7 +717,7 @@ public void deleteGroup(String fld, String txt) throws Exception {
* @return
* @throws Exception
*/
private List<Pair<String,Pair<Document, List<CategoryPath>>>> buildIndexDocument(String schemaDir, Element metadata, String id,
private List<Pair<String,Pair<Document, Collection<CategoryPath>>>> buildIndexDocument(String schemaDir, Element metadata, String id,
List<Element> moreFields, MetadataType metadataType, String title,
boolean group) throws Exception
{
Expand Down Expand Up @@ -767,7 +753,7 @@ private List<Pair<String,Pair<Document, List<CategoryPath>>>> buildIndexDocument
List<Element> documentElements = xmlDoc.getContent();
Collection<Field> multilingualSortFields = findMultilingualSortElements(documentElements);

List<Pair<String, Pair<Document, List<CategoryPath>>>> documents = new ArrayList<Pair<String, Pair<Document, List<CategoryPath>>>>();
List<Pair<String, Pair<Document, Collection<CategoryPath>>>> documents = new ArrayList<Pair<String, Pair<Document, Collection<CategoryPath>>>>();
for( Element doc : documentElements ) {
// add _id field
SearchManager.addField(doc, LuceneIndexField.ID, id, true, true);
Expand Down Expand Up @@ -1391,10 +1377,10 @@ public boolean rebuildIndex(ServiceContext context, boolean xlinks, boolean rese
* @param multilingualSortFields
* @return
*/
private Pair<Document, List<CategoryPath>> newDocument(Element xml, Collection<Field> multilingualSortFields)
private Pair<Document, Collection<CategoryPath>> newDocument(Element xml, Collection<Field> multilingualSortFields)
{
Document doc = new Document();
List<CategoryPath> categories = new ArrayList<CategoryPath>();
Collection<CategoryPath> categories = new HashSet<CategoryPath>();


for (Field field : multilingualSortFields) {
Expand Down Expand Up @@ -1480,12 +1466,16 @@ private Pair<Document, List<CategoryPath>> newDocument(Element xml, Collection<F

// Add value to the taxonomy
// TODO : Add all facets whatever the types
if(_luceneConfig.getTaxonomy().get("hits").get(name) != null) {
for (Map<String, FacetConfig> facets : _luceneConfig.getTaxonomy().values()) {
if (facets.containsKey(name)) {
if(Log.isDebugEnabled(Geonet.INDEX_ENGINE)) {
Log.debug(Geonet.INDEX_ENGINE, "Add category path: " + name + " with " + string);
}
categories.add(new CategoryPath(name, string));

break;
}
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/fao/geonet/kernel/search/index/LuceneIndexLanguageTracker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ synchronized void withWriter(Function function) throws CorruptIndexException, IO
}
}

public synchronized void addDocument(String language, Document doc, List<CategoryPath> categories)
public synchronized void addDocument(String language, Document doc, Collection<CategoryPath> categories)
throws CorruptIndexException, LockObtainFailedException, IOException {
lazyInit();
if (Log.isDebugEnabled(Geonet.INDEX_ENGINE)) {
Expand Down
7 changes: 2 additions & 5 deletions core/src/main/java/org/fao/geonet/kernel/search/index/TaxonomyIndexTracker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.*;

import org.apache.lucene.store.Directory;
import org.fao.geonet.utils.IO;
Expand Down Expand Up @@ -88,7 +85,7 @@ TaxonomyReader acquire() throws IOException {
}


void addDocument(Document doc, List<CategoryPath> categories) {
void addDocument(Document doc, Collection<CategoryPath> categories) {
try {
FacetFields facetFields = new FacetFields(taxonomyWriter);
facetFields.addFields(doc, categories);
Expand Down
18 changes: 7 additions & 11 deletions core/src/main/java/org/fao/geonet/util/FileCopyMgr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,27 @@
import java.io.File;
import java.io.IOException;

import com.google.common.collect.FluentIterable;
import com.google.common.io.Files;
import org.fao.geonet.utils.BinaryFile;
import org.fao.geonet.utils.Log;

import org.apache.commons.io.FileUtils;
import org.fao.geonet.constants.Geonet;

public class FileCopyMgr {
public static void copyFiles(String strPath, String dstPath) throws IOException {

File src = new File(strPath);
File dest = new File(dstPath);
BinaryFile.copy(src, dest);
}

/**
* Remove directory or file recursively
*
* @param dir
* @throws IOException
*/
public static void removeDirectoryOrFile(File dir) {
try {
FileUtils.deleteDirectory(dir);
} catch (IOException e) {
Log.warning(Geonet.GEONETWORK, "A failure occured while trying to delete: " + dir.getAbsolutePath(), e);
final FluentIterable<File> files = Files.fileTreeTraverser().postOrderTraversal(dir);
for (File file : files) {
if (!file.delete()) {
Log.warning(Geonet.MEF, "Unable to delete file: "+file);
}
}
}
}
Expand Down
Binary file modified core/src/test/resources/org/fao/geonet/kernel/mef/mef2-example-2md.zip
View file
Binary file not shown.
30 changes: 18 additions & 12 deletions services/src/main/java/org/fao/geonet/services/main/SearchSuggestion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@
import jeeves.interfaces.Service;
import jeeves.server.ServiceConfig;
import jeeves.server.context.ServiceContext;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.WildcardQuery;
import org.fao.geonet.exceptions.BadParameterEx;
import org.fao.geonet.kernel.search.IndexAndTaxonomy;
import org.fao.geonet.kernel.search.LuceneConfig;
import org.fao.geonet.kernel.search.index.GeonetworkMultiReader;
import org.fao.geonet.utils.Log;
import org.fao.geonet.Util;
import org.apache.commons.lang.StringUtils;
Expand All @@ -35,13 +44,13 @@
import org.fao.geonet.kernel.search.LuceneSearcher;
import org.fao.geonet.kernel.search.SearchManager;
import org.fao.geonet.kernel.search.SearchManager.TermFrequency;
import org.fao.geonet.utils.Xml;
import org.jdom.Attribute;
import org.jdom.Content;
import org.jdom.Element;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.io.IOException;
import java.util.*;

/**
* Return a list of suggestion for a field. The values could be filtered and
Expand Down Expand Up @@ -81,6 +90,7 @@ public class SearchSuggestion implements Service {
static final String CONFIG_PARAM_MAX_NUMBER_OF_TERMS = "max_number_of_terms";
static final String CONFIG_PARAM_DEFAULT_SEARCH_FIELD = "default_search_field";
static final String CONFIG_PARAM_THRESHOLD = PARAM_THRESHOLD;
private static final String SUMMARY_FACET_CONFIG_KEY = "suggestions";

/**
* Max number of term's values to look in the index. For large catalogue
Expand Down Expand Up @@ -196,12 +206,9 @@ public Element exec(Element params, ServiceContext context)
// The main advantage is that only values from records visible to the
// user are returned, because the search filter the results first.
if (origin.equals("") || origin.equals(RECORDS_FIELD_VALUES)) {
LuceneSearcher searcher = (LuceneSearcher) sm.newSearcher(
SearchManager.LUCENE, Geonet.File.SEARCH_LUCENE);

listOfSuggestions.addAll(searcher.getSuggestionForFields(
context, fieldName, searchValue, _config, maxNumberOfTerms,
threshold));
LuceneSearcher searcher = (LuceneSearcher) sm.newSearcher(SearchManager.LUCENE, Geonet.File.SEARCH_LUCENE);

searcher.getSuggestionForFields(context, fieldName, searchValue, _config, maxNumberOfTerms, threshold, listOfSuggestions);
}
// No values found from the index records field value ...
if (origin.equals(INDEX_TERM_VALUES)
Expand All @@ -222,8 +229,7 @@ public Element exec(Element params, ServiceContext context)

// Starts with element first
if (sortBy.equalsIgnoreCase(SORT_BY_OPTION.STARTSWITHFIRST.toString())) {
Collections.sort(listOfSuggestions, new StartsWithComparator(
searchValueWithoutWildcard));
Collections.sort(listOfSuggestions, new StartsWithComparator(searchValueWithoutWildcard));
} else if (sortBy.equalsIgnoreCase(SORT_BY_OPTION.ALPHA.toString())) {
// Sort by alpha and frequency
Collections.sort(listOfSuggestions);
Expand Down
Loading

0 comments on commit 731cd34

Please sign in to comment.