Author: bpoussin Date: 2010-12-22 04:40:56 +0100 (Wed, 22 Dec 2010) New Revision: 642 Url: http://nuiton.org/repositories/revision/wikitty/642 Log: - search case unsensitive and accent unsensitive work for like and unlike Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java trunk/wikitty-solr-impl/src/main/resources/schema.xml trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java 2010-12-21 15:34:09 UTC (rev 641) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java 2010-12-22 03:40:56 UTC (rev 642) @@ -285,7 +285,7 @@ element2solr += WikittySolrConstant.SUFFIX_STRING_FULLTEXT; break; case ToLowerCase: - element2solr += WikittySolrConstant.SUFFIX_STRING_FULLTEXT; + element2solr += WikittySolrConstant.SUFFIX_STRING_LOWERCASE; break; } } Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2010-12-21 15:34:09 UTC (rev 641) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2010-12-22 03:40:56 UTC (rev 642) @@ -62,6 +62,7 @@ import org.nuiton.wikitty.search.Search; import java.io.File; +import java.util.Collections; import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.WikittyUtil; @@ -244,7 +245,6 @@ // Copy old field value SolrDocument found = SolrUtil.findById(solrServer, id); if (found != null) { - // FIXME poussin 20101209 a finir en ne copiant que le necessaire voir javadoc du package doc = SolrUtil.copySolrDocument(found, true, fieldToCopyPattern); solrResource.addDoc(id, doc); } else { @@ -263,7 +263,7 @@ Collection<String> includedChildNodeIds = getIncludedNodeIds(id); Collection<String> excludedChildNodeIds = getExcludedNodeIds(id); - // Find all node contain child + // Find all node contain is as attachment SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + SUFFIX_WIKITTY + ":" + id); @@ -389,31 +389,56 @@ if (w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { Set<String> attachments = WikittyTreeNodeHelper.getAttachment(w); - reindexChildTreeNode.putIncludedAttachments(id, attachments); + if (attachments == null) { + attachments = Collections.emptySet(); + } - // Search deleted children + // + // recherche des anciennes attachments + // + Collection oldAttachments = null; + Set<String> oldAttachmentsCopy; + + // Search deleted attachments SolrDocument treeNodeDoc = SolrUtil.findById(solrServer, id); if (treeNodeDoc != null) { - Collection oldAttachments = treeNodeDoc.getFieldValues( + oldAttachments = treeNodeDoc.getFieldValues( WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + SUFFIX_WIKITTY); - if (oldAttachments != null) { - // if no more children, remove all old children - if(attachments == null) { - reindexChildTreeNode.putExcludedAttachments(id, oldAttachments); - } else { - // exclude only the removed children - for (Object oldAttachment : oldAttachments) { - if(!attachments.contains(oldAttachment)) { - reindexChildTreeNode.putExcludedAttachment(id,(String) oldAttachment); - } - } - } - } } + // make copy to modify it + if (oldAttachments == null) { + oldAttachments = Collections.emptySet(); + } + oldAttachmentsCopy = new HashSet<String>(oldAttachments); - // Get new parent id (may be the same old parent) - String parentId = WikittyTreeNodeHelper.getParent(w); - reindexChildTreeNode.putParent(id, parentId); + + + // les attachments seulement dans old doivent etre exclus + oldAttachmentsCopy.removeAll(attachments); + reindexChildTreeNode.putExcludedAttachments(id, oldAttachmentsCopy); + + if (w.getDirty().contains(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT) + || null == WikittyTreeNodeHelper.getParent(w) + || WikittyUtil.versionGreaterThan("1", w.getVersion())) { + // si le pere a change + // ou qu'il est null (creation d'un nouvel arbre) + // ou que l'objet n'a jamais ete sauve 1 > version + // il faut indexer les attachments + + // il faut reindexer tous les attachments + reindexChildTreeNode.putIncludedAttachments(id, attachments); + + // Get new parent id (may be the same old parent) + String parentId = WikittyTreeNodeHelper.getParent(w); + reindexChildTreeNode.putParent(id, parentId); + } else { + // les attachments seulement dans les nouveaux doivent etre inclus + Set<String> attachmentsCopy = new HashSet<String>(attachments); + attachmentsCopy.removeAll(oldAttachments); + reindexChildTreeNode.putIncludedAttachments(id, attachmentsCopy); + } + + } // Index Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java 2010-12-21 15:34:09 UTC (rev 641) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java 2010-12-22 03:40:56 UTC (rev 642) @@ -65,8 +65,6 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import org.nuiton.util.ApplicationConfig; -import org.nuiton.util.CollectionUtil; -import org.nuiton.util.StringUtil; import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; @@ -257,6 +255,14 @@ return result; } + /** + * Reindexe les noeuds passe en parametre et reindexe automatiquement leurs + * sous noeuds + * + * @param solrResource + * @param treeNodeToIndex + * @throws Exception + */ protected void reindexTreeNode(SolrResource solrResource, LinkedHashSet<String> treeNodeToIndex) throws Exception { LinkedList<String> todo = new LinkedList<String>(treeNodeToIndex); @@ -341,6 +347,7 @@ for (String id : ids) { SolrDocument doc = SolrUtil.findById(solrServer, id); if (doc.containsKey(TREENODE_ROOT)) { + // child are automaticaly done in reindexTreeNode method treeNodeToIndex.add(id); } solrResource.deleteDoc(id); Modified: trunk/wikitty-solr-impl/src/main/resources/schema.xml =================================================================== --- trunk/wikitty-solr-impl/src/main/resources/schema.xml 2010-12-21 15:34:09 UTC (rev 641) +++ trunk/wikitty-solr-impl/src/main/resources/schema.xml 2010-12-22 03:40:56 UTC (rev 642) @@ -58,6 +58,7 @@ sortMissingLast="true"> <analyzer> <!-- no type to indicated that used it for both type: index and query --> <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.ASCIIFoldingFilterFactory"/> <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> @@ -75,6 +76,7 @@ positionIncrementGap="100"> <analyzer type="index"> <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.ASCIIFoldingFilterFactory"/> <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords_fr.txt" @@ -82,7 +84,8 @@ /> <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" - catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/> + catenateNumbers="1" catenateAll="0" splitOnCaseChange="1" + preserveOriginal="1"/> <filter class="solr.LowerCaseFilterFactory"/> <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> @@ -90,13 +93,15 @@ </analyzer> <analyzer type="query"> <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.ASCIIFoldingFilterFactory"/> <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords_fr.txt"/> <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" - catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/> + catenateNumbers="0" catenateAll="0" splitOnCaseChange="1" + preserveOriginal="1"/> <filter class="solr.LowerCaseFilterFactory"/> <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java 2010-12-21 15:34:09 UTC (rev 641) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java 2010-12-22 03:40:56 UTC (rev 642) @@ -540,9 +540,32 @@ { // avec accent specifiquement sur le champs name mais sans accent pour la recherche Criteria criteria = Search.query().eq( - WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME,"ceca").criteria(); + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME + + WikittySolrConstant.SUFFIX_STRING + + WikittySolrConstant.SUFFIX_STRING_FULLTEXT, "ceca").criteria(); PagedResult<String> result = ws.findAllByCriteria(null, criteria); Assert.assertEquals(1, result.size()); } + { + // avec accent specifiquement sur le champs name + // mais sans accent pour la recherche + // et et majuscule + Criteria criteria = Search.query().eq( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME + + WikittySolrConstant.SUFFIX_STRING + + WikittySolrConstant.SUFFIX_STRING_FULLTEXT, "CECA").criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + Assert.assertEquals(1, result.size()); + } + { + System.out.println("#######"); + // avec accent specifiquement sur le champs name + // mais sans accent pour la recherche + // et et majuscule + Criteria criteria = Search.query().like( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME, "CECA").criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + Assert.assertEquals(1, result.size()); + } } } Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java 2010-12-21 15:34:09 UTC (rev 641) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java 2010-12-22 03:40:56 UTC (rev 642) @@ -506,4 +506,19 @@ sum = sum(node32); assertEquals(1, sum); } + + // verifier que l'indexation de l'arbre fonctionne bien et la reindexation aussi + @Test + public void testReindexation() throws Exception { +// FIXME poussin 20101222 a faire + // creer deux arbres (3 noeud) + // mettre 3 attachments par noeud (les memes) dans les 2 arbres + + // enlever un attachment par noeud et supprime le noeud intermediaire + // en meme temps et store les arbres + + // verifier que tout est bien reindexe et que le dernier noeud est bien + // devenu un noeud root + } + }