Author: echatellier Date: 2011-06-28 16:47:28 +0200 (Tue, 28 Jun 2011) New Revision: 982 Url: http://nuiton.org/repositories/revision/wikitty/982 Log: Ajout du support de la facetisation sur bobo Modified: trunk/wikitty-lucene/src/main/java/org/nuiton/wikitty/storage/lucene/WikittySearchEngineLucene.java Modified: trunk/wikitty-lucene/src/main/java/org/nuiton/wikitty/storage/lucene/WikittySearchEngineLucene.java =================================================================== --- trunk/wikitty-lucene/src/main/java/org/nuiton/wikitty/storage/lucene/WikittySearchEngineLucene.java 2011-06-28 12:34:17 UTC (rev 981) +++ trunk/wikitty-lucene/src/main/java/org/nuiton/wikitty/storage/lucene/WikittySearchEngineLucene.java 2011-06-28 14:47:28 UTC (rev 982) @@ -25,6 +25,7 @@ package org.nuiton.wikitty.storage.lucene; +import static org.nuiton.wikitty.storage.lucene.WikittyLuceneConstants.LUCENE_DEFAULT_FIELD; import static org.nuiton.wikitty.storage.lucene.WikittyLuceneConstants.LUCENE_ID; import static org.nuiton.wikitty.storage.lucene.WikittyLuceneConstants.TREENODE_ATTACHED; import static org.nuiton.wikitty.storage.lucene.WikittyLuceneConstants.TREENODE_ATTACHED_ALL; @@ -56,6 +57,8 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; @@ -64,11 +67,9 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.Version; @@ -77,7 +78,6 @@ import org.nuiton.wikitty.WikittyConfigOption; import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.WikittyUtil; -import org.nuiton.wikitty.entities.FieldType; import org.nuiton.wikitty.entities.FieldType.TYPE; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyTreeNode; @@ -92,6 +92,19 @@ import org.nuiton.wikitty.storage.WikittyExtensionStorage; import org.nuiton.wikitty.storage.WikittySearchEngine; +import com.browseengine.bobo.api.BoboBrowser; +import com.browseengine.bobo.api.BoboIndexReader; +import com.browseengine.bobo.api.Browsable; +import com.browseengine.bobo.api.BrowseException; +import com.browseengine.bobo.api.BrowseFacet; +import com.browseengine.bobo.api.BrowseHit; +import com.browseengine.bobo.api.BrowseRequest; +import com.browseengine.bobo.api.BrowseResult; +import com.browseengine.bobo.api.FacetAccessible; +import com.browseengine.bobo.api.FacetSpec; +import com.browseengine.bobo.facets.FacetHandler; +import com.browseengine.bobo.facets.impl.SimpleFacetHandler; + /** * Wikitty search engine implementation based on Lucene for storage and search * engine and bobo for facetting support. @@ -225,40 +238,6 @@ /* * @see org.nuiton.wikitty.storage.WikittySearchEngine#store(org.nuiton.wikitty.services.WikittyTransaction, java.util.Collection, boolean) - * - @Override - public void store(WikittyTransaction transaction, - Collection<Wikitty> wikitties, boolean force) throws WikittyException { - - // TODO manage transaction - - IndexWriter writer = null; - try { - // must be instancied every time (single instance per writer) - IndexWriterConfig indexWriterConfig = new IndexWriterConfig(WIKITTY_LUCENE_VERSION, indexAnalyzer); - writer = new IndexWriter(indexDirectory, indexWriterConfig); - - for (Wikitty w : wikitties) { - Document wDoc = convertWikittyToDoc(w); - writer.addDocument(wDoc); - } - - writer.commit(); - - } catch (IOException ex) { - try { - writer.rollback(); - } catch (IOException e) { - throw new WikittyException("Can't rollback index", ex); - } - throw new WikittyException("Can't store to index", ex); - } finally { - IOUtils.closeQuietly(writer); - } - }*/ - - /* - * @see org.nuiton.wikitty.storage.WikittySearchEngine#store(org.nuiton.wikitty.services.WikittyTransaction, java.util.Collection, boolean) */ @Override public void store(WikittyTransaction transaction, @@ -585,9 +564,11 @@ addedDocs.put(attId, doc); } doc.removeField(TREENODE_ATTACHED + treeNodeId); + doc.removeField(TREENODE_ATTACHED_ALL); } } } + for (String treeNodeId : attachmentInTree.getAdded().keySet()) { Collection<String> treeNodeParents = null; Document treeNodeDoc = addedDocs.get(treeNodeId); @@ -603,7 +584,7 @@ String[] treeNodeParent = doc.getValues(TREENODE_PARENTS); treeNodeParents = Arrays.asList(treeNodeParent); } else { - log.error("SolR doc not found in Transaction or in tree." + log.error("Lucene doc not found in Transaction or in tree." + "This is a bug !!!"); } // add tree indexation on all attachments for this treeNodeId @@ -636,6 +617,7 @@ for (String id : treeNodeParents) { //doc.addField(TREENODE_ATTACHED + treeNodeId, id); doc.add(new Field(TREENODE_ATTACHED + treeNodeId, id, Store.YES, Index.NOT_ANALYZED)); + doc.add(new Field(TREENODE_ATTACHED_ALL, id, Store.YES, Index.NOT_ANALYZED)); } } } @@ -725,9 +707,11 @@ // wikitty extension String extAsString = StringUtils.join(w.getExtensionNames(), ","); - Field luceneExtField = new Field(WikittyLuceneConstants.LUCENE_EXTENSIONS, + if (!extAsString.isEmpty()) { + Field luceneExtField = new Field(WikittyLuceneConstants.LUCENE_EXTENSIONS, extAsString, Store.YES, Index.NOT_ANALYZED); - document.add(luceneExtField); + document.add(luceneExtField); + } StringBuffer allAsText = new StringBuffer(); // all other wikitty fields @@ -767,7 +751,7 @@ } // le champ text n'est pas stocké - Field luceneFullTextField = new Field(WikittyLuceneConstants.LUCENE_DEFAULT_FIELD, + Field luceneFullTextField = new Field(LUCENE_DEFAULT_FIELD, allAsText.toString(), Store.NO, Index.ANALYZED); document.add(luceneFullTextField); @@ -815,15 +799,15 @@ * @see org.nuiton.wikitty.storage.WikittySearchEngine#findAllByCriteria(org.nuiton.wikitty.services.WikittyTransaction, org.nuiton.wikitty.search.Criteria) */ @Override - public PagedResult<String> findAllByCriteria( - WikittyTransaction transaction, Criteria criteria) throws WikittyException { + public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, + Criteria criteria) throws WikittyException { // TODO manage transaction PagedResult<String> pagedResult = null; - IndexSearcher searcher = null; + IndexReader indexReader = null; try { - searcher = new IndexSearcher(indexDirectory); + indexReader = IndexReader.open(indexDirectory); // get lucene query Restriction restriction = criteria.getRestriction(); @@ -859,11 +843,95 @@ int firstIndex = criteria.getFirstIndex(); int endIndex = criteria.getEndIndex(); if (endIndex < 0) { - endIndex = searcher.maxDoc(); + endIndex = indexReader.maxDoc(); // indexSearcher.maxDoc(); } + + // facetting support : bobo + List<String> facetFields = criteria.getFacetField(); + List<FacetHandler<?>> facetHandlers = new ArrayList<FacetHandler<?>>(); + if (facetFields != null) { + for (String facetField : facetFields) { + SimpleFacetHandler facetHandler = new SimpleFacetHandler(facetField); + facetHandlers.add(facetHandler); + } + } + BoboIndexReader boboReader = BoboIndexReader.getInstance(indexReader, facetHandlers); + // get browse request + BrowseRequest br = new BrowseRequest(); + br.setOffset(firstIndex); + br.setCount(endIndex - firstIndex); + br.setQuery(query); + if (sortFields.isEmpty()) { + SortField[] sortFieldsA = sortFields.toArray(new SortField[sortFields.size()]); + br.setSort(sortFieldsA); + } + + // manage facets + //Map<String, String> facetQueryToName = new HashMap<String, String>(); + + //List<Criteria> facetCriterias = criteria.getFacetCriteria(); + if (facetFields != null) { + for (String facetField : facetFields) { + FacetSpec facetSpec = new FacetSpec(); + //facetSpec.setOrderBy(FacetSortSpec.OrderHitsDesc); + facetSpec.setMinHitCount(1); + br.setFacetSpec(facetField, facetSpec); + } + } + + /*// manage criteria facets + List<Criteria> facetCriterias = criteria.getFacetCriteria(); + if (facetCriterias != null) { + for (Criteria facetCriteria : facetCriterias) { + Query facetQuery = restriction2Lucene.toLucene(facetCriteria.getRestriction()); + } + }*/ + + // perform request + Browsable browser = new BoboBrowser(boboReader); + BrowseResult result = browser.browse(br); + + // manage request results (real results) + int pagedResultCount = result.getNumHits(); + BrowseHit[] hits = result.getHits(); + List<String> results = new ArrayList<String>(); + for (BrowseHit hit : hits) { + //if (currentDocIndex >= firstIndex && currentDocIndex <= endIndex) { + int luceneId = hit.getDocid(); + Document document = indexReader.document(luceneId); + String wikittyId = document.get(WikittyLuceneConstants.LUCENE_ID); + results.add(wikittyId); + //} + } + + // manage result facets + Map<String, FacetAccessible> resultFacets = result.getFacetMap(); + Map<String, List<FacetTopic>> pagedResultFacets = new HashMap<String, List<FacetTopic>>(); + for (Map.Entry<String, FacetAccessible> resultFacet : resultFacets.entrySet()) { + String facetName = resultFacet.getKey(); + FacetAccessible facetValue = resultFacet.getValue(); + + List<FacetTopic> facetTopics = new ArrayList<FacetTopic>(); + List<BrowseFacet> browseFacets = facetValue.getFacets(); + for (BrowseFacet browseFacet : browseFacets) { + FacetTopic topic = new FacetTopic(facetName, browseFacet.getValue(), + browseFacet.getFacetValueHitCount()); + facetTopics.add(topic); + } + + pagedResultFacets.put(facetName, facetTopics); + + // clean up. This is to free up the resources used in facet counting + // and this is new for bobo 2.5. Very important to call. + facetValue.close(); + } + + pagedResult = new PagedResult<String>(firstIndex, pagedResultCount, + query.toString(), pagedResultFacets, results); + // execute search query - List<String> results = new ArrayList<String>(); + /*List<String> results = new ArrayList<String>(); int totalHitCount = 0; if (endIndex == 0) { // il faut un cas special pour le 0, sinon lucene rale. @@ -896,15 +964,17 @@ } totalHitCount = topDocs.totalHits; - } - - pagedResult = new PagedResult<String>(firstIndex, totalHitCount, + + pagedResult = new PagedResult<String>(firstIndex, totalHitCount, query.toString(), null, results); + }*/ } catch (IOException ex) { throw new WikittyException("Can't search on index", ex); + } catch (BrowseException ex) { + throw new WikittyException("Can't search on index", ex); } finally { - IOUtils.closeQuietly(searcher); + IOUtils.closeQuietly(indexReader); } return pagedResult; @@ -1055,12 +1125,15 @@ Document doc = searcher.doc(scoreDoc.doc); String id = doc.getFieldable(LUCENE_ID).stringValue(); - String parentId = doc.getFieldable( - WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT).stringValue(); + String parentId = null; + Fieldable fieldable = doc.getFieldable(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT); + if (fieldable != null) { + parentId = fieldable.stringValue(); + } int nb = counts.containsKey(id) ? counts.get(id) : 0; TreeNodeResult<String> child = new TreeNodeResult<String>(id, nb); allTreeNodeResult.put(id, child); - childParent.put(id, parentId); + childParent.put(id, parentId); } // construction de l'arbre avant de le retourner for(Map.Entry<String, TreeNodeResult<String>> e : allTreeNodeResult.entrySet()) {