Author: bpoussin Date: 2011-01-31 13:13:07 +0100 (Mon, 31 Jan 2011) New Revision: 699 Url: http://nuiton.org/repositories/revision/wikitty/699 Log: Evolution #1277: Rewrite solr indexer to minimize solr server call Evolution #1278: remove tree method in WikittyService and replace them with one method findTreeNode Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/TreeNodeResult.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/AttachmentInTree.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Removed: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyTree.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrResource.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrUtil.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySolrConstant.java trunk/wikitty-solr-impl/src/main/resources/schema.xml trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrTest.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyProxy.java 2011-01-31 12:13:07 UTC (rev 699) @@ -47,10 +47,11 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; +import org.nuiton.wikitty.entities.WikittyTreeNode; +import org.nuiton.wikitty.search.TreeNodeResult; /** * Wikitty proxy is used to transform wikitty object used by {@link WikittyService} @@ -533,6 +534,176 @@ return result; } + /** + * Recupere une portion d'arbre a partir de l'id passer en parametre. L'id + * doit etre celui d'un WikittyTreeNode. Ce WikittyTreeNode est alors le + * root de l'arbre retourne. + * + * Return WikittyTreeNode in result + * + * @param wikittyId root + * @param depth profondeur de noeud a recuperer + * @param count vrai si l'on veut le nombre de piece attaches sur le noeud (piece des enfants compris) + * @param filter filter pour compter les pieces attachees + * @return + * + * @since 3.1 + */ + public TreeNodeResult<WikittyTreeNode> findTreeNode( + String wikittyId, int depth, boolean count, Criteria filter) { + long start = TimeTrace.getTime(); + TreeNodeResult<String> resultId = wikittyService.findTreeNode( + securityToken, wikittyId, depth, count, filter); + + RetrieveIdVisitor retrieveIdVisitor = new RetrieveIdVisitor(); + resultId.acceptVisitor(retrieveIdVisitor); + + List<Wikitty> wikitties = + wikittyService.restore(securityToken, retrieveIdVisitor.getIds()); + + IdToTreeNodeConverter converter = new IdToTreeNodeConverter( + securityToken, wikittyService, wikitties); + + ConvertTreeVisitor<WikittyTreeNode> convertVisitor = + new ConvertTreeVisitor<WikittyTreeNode>(converter); + + resultId.acceptVisitor(convertVisitor); + + TreeNodeResult<WikittyTreeNode> result = convertVisitor.getTree(); + timeTrace.add(start, "findTreeNode"); + return result; + } + + /** + * Used to collect all node id + * @since 3.1 + */ + static private class RetrieveIdVisitor implements TreeNodeResult.Visitor<String> { + + protected List<String> ids = new ArrayList<String>(); + + public List<String> getIds() { + return ids; + } + + @Override + public boolean visitEnter(TreeNodeResult<String> node) { + String id = node.getObject(); + ids.add(id); + return true; + } + + @Override + public boolean visitLeave(TreeNodeResult<String> node) { + return true; + } + } + + /** + * Converti un id en son object WikittyTreeNode + * @since 3.1 + */ + static private class IdToTreeNodeConverter implements ConvertTreeVisitor.Converter<String, WikittyTreeNode> { + protected Map<String, Wikitty> wikitties = new HashMap<String, Wikitty>(); + protected String securityToken; + protected WikittyService wikittyService; + public IdToTreeNodeConverter(String securityToken, + WikittyService wikittyService, List<Wikitty> wikitties) { + this.securityToken = securityToken; + this.wikittyService = wikittyService; + + for (Wikitty w : wikitties) { + this.wikitties.put(w.getId(), w); + } + } + + @Override + public WikittyTreeNode convert(String id) { + Wikitty w = wikitties.get(id); + WikittyTreeNode result = WikittyUtil.newInstance( + securityToken, wikittyService, WikittyTreeNode.class, w); + + return result; + } + } + + /** + * Parcours un TreeNodeResult et en fait une copie en modifiant le type + * d'objet stocker dans le noeud grace a un converter, si le converter + * est null une exception est levee + * + * @param <TARGET> le type d'objet pour le nouvel arbre + * @since 3.1 + */ + static private class ConvertTreeVisitor<TARGET> implements TreeNodeResult.Visitor<String> { + + static private interface Converter<SOURCE, TARGET> { + public TARGET convert(SOURCE o); + } + protected Converter<String, TARGET> converter; + protected TreeNodeResult<TARGET> tree = null; + protected LinkedList<TreeNodeResult<TARGET>> stack = + new LinkedList<TreeNodeResult<TARGET>>(); + + public ConvertTreeVisitor(Converter<String, TARGET> converter) { + this.converter = converter; + if (converter == null) { + throw new IllegalArgumentException("Converter can't be null"); + } + } + + public TreeNodeResult<TARGET> getTree() { + return tree; + } + + @Override + public boolean visitEnter(TreeNodeResult<String> node) { + String id = node.getObject(); + int count = node.getAttCount(); + + TARGET object = converter.convert(id); + TreeNodeResult<TARGET> newNode = new TreeNodeResult<TARGET>( + object, count); + + TreeNodeResult<TARGET> parent = stack.peekLast(); + if (parent == null) { + // le premier noeud, donc le root a retourner plus tard + tree = newNode; + } else { + parent.add(newNode); + } + + stack.offerLast(newNode); + + return true; + } + + @Override + public boolean visitLeave(TreeNodeResult<String> node) { + stack.pollLast(); + return true; + } + } + + /** + * Return just wikitty Id in result + * @param wikittyId + * @param depth + * @param count + * @param filter + * @return + * @since 3.1 + */ + public TreeNodeResult<String> findAllIdTreeNode( + String wikittyId, int depth, boolean count, Criteria filter) { + long start = TimeTrace.getTime(); + TreeNodeResult<String> result = wikittyService.findTreeNode( + securityToken, wikittyId, depth, count, filter); + + timeTrace.add(start, "findAllIdTreeNode"); + return result; + } + public <E extends BusinessEntity> E findByCriteria(Class<E> clazz, Criteria criteria) { long start = TimeTrace.getTime(); BusinessEntityImpl sample = @@ -562,14 +733,6 @@ return wikitty; } - public WikittyTree restoreTree(String wikittyId) { - long start = TimeTrace.getTime(); - WikittyTree result = wikittyService.restoreTree(securityToken, wikittyId); - - timeTrace.add(start, "restoreTree"); - return result; - } - /** * Delete specified tree node and all sub nodes. * @@ -584,60 +747,63 @@ return result; } - public <E extends BusinessEntity> Map.Entry<E, Integer> restoreNode( - Class<E> clazz, String wikittyId, Criteria filter) { - Entry<E, Integer> result = restoreNode(clazz, wikittyId, filter, false); - return result; - } + // Next method are removed, but if necessary they can be reimplanted with + // call to findTreeNode methode - public <E extends BusinessEntity> Map.Entry<E, Integer> restoreNode( - Class<E> clazz, String wikittyId, Criteria filter, boolean checkExtension) { - long start = TimeTrace.getTime(); - Map.Entry<E, Integer> result = null; +// public <E extends BusinessEntity> Map.Entry<E, Integer> restoreNode( +// Class<E> clazz, String wikittyId, Criteria filter) { +// Entry<E, Integer> result = restoreNode(clazz, wikittyId, filter, false); +// return result; +// } +// +// public <E extends BusinessEntity> Map.Entry<E, Integer> restoreNode( +// Class<E> clazz, String wikittyId, Criteria filter, boolean checkExtension) { +// long start = TimeTrace.getTime(); +// Map.Entry<E, Integer> result = null; +// +// Map.Entry<String, Integer> node = wikittyService.restoreNode( +// securityToken, wikittyId, filter); +// if (node != null) { +// E bean = restore(clazz, wikittyId, checkExtension); +// if (bean != null) { +// result = new HashMap.SimpleEntry<E, Integer>(bean, node.getValue()); +// } +// } +// +// timeTrace.add(start, "restoreNode"); +// return result; +// } +// +// public <E extends BusinessEntity> Map<E, Integer> findTreeNode( +// Class<E> clazz, String wikittyId, Criteria filter) { +// Map<E, Integer> result = findTreeNode(clazz, wikittyId, filter, false); +// return result; +// } +// +// public <E extends BusinessEntity> Map<E, Integer> findTreeNode( +// Class<E> clazz, String wikittyId, Criteria filter, boolean checkExtension) { +// long start = TimeTrace.getTime(); +// +// Map<E, Integer> convertedResult = null; +// +// Map<String, Integer> result = wikittyService.findTreeNode( +// securityToken, wikittyId, filter); +// +// if(result != null) { +// List<String> ids = new ArrayList<String>(result.keySet()); +// List<E> keys = restore(clazz, ids, checkExtension); +// +// convertedResult = new LinkedHashMap<E, Integer>(); +// for (E e : keys) { +// Integer size = result.get(e.getWikittyId()); +// convertedResult.put(e, size); +// } +// } +// +// timeTrace.add(start, "findTreeNode"); +// return convertedResult; +// } - Map.Entry<String, Integer> node = wikittyService.restoreNode( - securityToken, wikittyId, filter); - if (node != null) { - E bean = restore(clazz, wikittyId, checkExtension); - if (bean != null) { - result = new HashMap.SimpleEntry<E, Integer>(bean, node.getValue()); - } - } - - timeTrace.add(start, "restoreNode"); - return result; - } - - public <E extends BusinessEntity> Map<E, Integer> restoreChildren( - Class<E> clazz, String wikittyId, Criteria filter) { - Map<E, Integer> result = restoreChildren(clazz, wikittyId, filter, false); - return result; - } - - public <E extends BusinessEntity> Map<E, Integer> restoreChildren( - Class<E> clazz, String wikittyId, Criteria filter, boolean checkExtension) { - long start = TimeTrace.getTime(); - - Map<E, Integer> convertedResult = null; - - Map<String, Integer> result = wikittyService.restoreChildren( - securityToken, wikittyId, filter); - - if(result != null) { - List<String> ids = new ArrayList<String>(result.keySet()); - List<E> keys = restore(clazz, ids, checkExtension); - - convertedResult = new LinkedHashMap<E, Integer>(); - for (E e : keys) { - Integer size = result.get(e.getWikittyId()); - convertedResult.put(e, size); - } - } - - timeTrace.add(start, "restoreChildren"); - return convertedResult; - } - public Wikitty restoreVersion(String wikittyId, String version) { long start = TimeTrace.getTime(); Wikitty result = wikittyService.restoreVersion( Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyService.java 2011-01-31 12:13:07 UTC (rev 699) @@ -33,8 +33,7 @@ import org.nuiton.wikitty.services.WikittyListener; import java.util.Collection; import java.util.List; -import java.util.Map; -import org.nuiton.wikitty.entities.WikittyTreeNode; +import org.nuiton.wikitty.search.TreeNodeResult; /** * Wikitty service. @@ -318,21 +317,6 @@ */ /** - * Retrieve all wikitties children (recursively) of an other one - * Wikitty reference by wikittyId MUST include the 'Node' extension - * - * @param securityToken security token - * @param wikittyId - * @return - * - * @deprecated cette methode retourne des wikitty or seul la methode restore - * doit le faire (centralisation de la restoration). Il faut donc que cette - * methode n'existe que sur le proxy et utilise les autres methodes - */ - @Deprecated - public WikittyTree restoreTree(String securityToken, String wikittyId); - - /** * Delete specified tree node and all sub nodes. * * @param securityToken security token @@ -342,36 +326,33 @@ public WikittyEvent deleteTree(String securityToken, String treeNodeId); /** - * Retrieve wikitty node with count. Wikitty reference by wikittyId MUST - * include the 'WikittyTreeNode' extension. + * Retrieve all node from wikittyId, this node is returned too. + * Returned wikitty must include the 'WikittyTreeNode' extension. * - * Count is number of attachment in subtree. If filter is not null only - * attachments that satisfy filter are counted + * depth ask the recursively level: + * <li> 0 return only wikittyId passed in argument + * <li> 1 return wikittyId passed in argument, and his children + * <li> ... + * <li> negative value return all node + * + * if count is true, integer in return map is number of attachment in subtree + * (recursively). If filter is not null only attachments that satisfy filter + * are counted. + * if count is false, all integer are fixed to 0 * * @param securityToken security token - * @param wikittyId - * @param filter + * @param wikittyId root node to begin + * @param depth depth of node returned + * @param count if true return count of attachment + * @param filter filter on attachment count * @return + * @since 3.1 */ - public Map.Entry<String, Integer> restoreNode( - String securityToken, String wikittyId, Criteria filter); + public TreeNodeResult<String> findTreeNode( + String securityToken, String wikittyId, + int depth, boolean count, Criteria filter); - /** - * Retrieve all wikitties children (no recursively) with count of an other one - * Wikitty reference by wikittyId MUST include the 'WikittyTreeNode' extension - * - * Count is number of attachment in subtree (recursively). If filter is not - * null only attachments that satisfy filter are counted - * - * @param securityToken security token - * @param wikittyId - * @param filter - * @return - */ - public Map<String, Integer> restoreChildren( - String securityToken, String wikittyId, Criteria filter); - /* * history */ Deleted: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyTree.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyTree.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyTree.java 2011-01-31 12:13:07 UTC (rev 699) @@ -1,62 +0,0 @@ -/* - * #%L - * Wikitty :: api - * * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2009 - 2010 CodeLutin, Benjamin Poussin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ -package org.nuiton.wikitty; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import org.nuiton.wikitty.entities.WikittyTreeNode; - -public class WikittyTree implements Serializable { - - /** serialVersionUID. */ - private static final long serialVersionUID = 5785244346524975096L; - - protected WikittyTreeNode node; - protected List<WikittyTree> children; - - public void setNode(WikittyTreeNode node) { - this.node = node; - } - - public WikittyTreeNode getNode() { - return node; - } - - public void addChild(WikittyTree tree) { - getChildren().add( tree ); - } - - public List<WikittyTree> getChildren() { - if ( children == null ) { - children = new ArrayList<WikittyTree>(); - } - return children; - } - - public String getName() { - return node.getName(); - } -} Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java 2011-01-31 12:13:07 UTC (rev 699) @@ -197,8 +197,20 @@ */ void clearField(String fqFieldName); + /** + * Return only used fieldNames + * @return + */ Set<String> fieldNames(); + /** + * Return all fieldName available in all extension + * @return + * + * @since 3.1 + */ + Set<String> getAllFieldNames(); + /** get the value of and field given its fqn */ Object getFqField(String fqFieldName); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java 2011-01-31 12:13:07 UTC (rev 699) @@ -345,6 +345,11 @@ } @Override + public Set<String> getAllFieldNames() { + return target.getAllFieldNames(); + } + + @Override public Object getFqField(String fqFieldName) { return target.getFqField(fqFieldName); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java 2011-01-31 12:13:07 UTC (rev 699) @@ -851,6 +851,18 @@ return fieldValue.keySet(); } + @Override + public Set<String> getAllFieldNames() { + Set<String> result = new HashSet<String>(); + for (WikittyExtension ext : getExtensions()) { + String extName = ext.getName() + WikittyUtil.FQ_FIELD_NAME_SEPARATOR; + for (String f : ext.getFieldNames()) { + result.add(extName + f); + } + } + return result; + } + /* * @see org.nuiton.wikitty.Wikitty#getFqField(java.lang.String) */ Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/TreeNodeResult.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/TreeNodeResult.java (rev 0) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/TreeNodeResult.java 2011-01-31 12:13:07 UTC (rev 699) @@ -0,0 +1,163 @@ +package org.nuiton.wikitty.search; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.List; +import java.util.Vector; +import javax.swing.tree.DefaultMutableTreeNode; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * L'iteration se fait en profondeur + * + * @author poussin + * @version $Revision$ + * @since 3.1 + * + * Last update: $Date$ + * by : $Author$ + */ +public class TreeNodeResult<T> extends DefaultMutableTreeNode implements Iterable<TreeNodeResult<T>> { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(TreeNodeResult.class); + + private static final long serialVersionUID = 31L; + + /** + * Visitor for TreeNodeResult + * @param <T> + */ + static public interface Visitor<T> { + /** + * + * @param node node to visit + * @return if true visit this element + */ + public boolean visitEnter(TreeNodeResult<T> node); + /** + * + * @param node node visited + * @return if true visit next element + */ + public boolean visitLeave(TreeNodeResult<T> node); + } + + protected int attCount; + + /** + * + * @param object L'id, le wikitty ou le BusinessEntity suivant le type de T + * @param attCount le nombre d'attachment pour ce noeud (avec les sous noeud) + */ + public TreeNodeResult(T object, int attCount) { + super(object); + this.attCount = attCount; + } + + /** + * Visite en profondeur de l'arbre, il est possible d'arreter la visite + * soit en entrant dans le noeud soit en sortant du noeud, si respectivement + * visitEnter ou visitLeave retourne false. + * + * @param visitor + */ + public boolean acceptVisitor(Visitor<T> visitor) { + if (visitor.visitEnter(this)) { + for (Enumeration e = children(); e.hasMoreElements();) { + TreeNodeResult<T> child = (TreeNodeResult<T>) e.nextElement(); + if (!child.acceptVisitor(visitor)) { + break; + } + } + } + boolean result = visitor.visitLeave(this); + return result; + } + + /** + * Get direct children of this node + * + * @return + */ + public List<TreeNodeResult<T>> getChildren() { + List<TreeNodeResult<T>> result; + if (children == null) { + result = Collections.emptyList(); + } else { + result = Collections.unmodifiableList( + new ArrayList<TreeNodeResult<T>>(children)); + } + return result; + } + + /** + * Iterate on all children or sub-children, in depth first + * @return + */ + @Override + public Iterator<TreeNodeResult<T>> iterator() { + Iterator<TreeNodeResult<T>> result = new Iterator<TreeNodeResult<T>>() { + + protected Enumeration enumDepth = TreeNodeResult.this.depthFirstEnumeration(); + + @Override + public boolean hasNext() { + return enumDepth.hasMoreElements(); + } + + @Override + public TreeNodeResult<T> next() { + TreeNodeResult<T> result = (TreeNodeResult<T>)enumDepth.nextElement(); + return result; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Not supported yet."); + } + }; + return result; + } + + /** + * Retourne l'objet associe avec ce noeud (id, wikitty ou BusinessEntity) + * @return l'objet associe avec ce noeud (id, wikitty ou BusinessEntity) + */ + public T getObject() { + return (T)getUserObject(); + } + + /** + * Return TreeNodeResult where object in TreeNodeResult equals child in + * parameter + * @param child + * @return + */ + public TreeNodeResult<T> getChild(T child) { + TreeNodeResult<T> result = null; + if (child != null) { + for (Enumeration e = children(); e.hasMoreElements();) { + TreeNodeResult<T> r = (TreeNodeResult<T>) e.nextElement(); + if (child.equals(r.getObject())) { + result = r; + break; + } + } + } + return result; + } + + /** + * Retourn le nombre d'attachment pour ce noeud (avec les sous noeud) + * @return + */ + public int getAttCount() { + return attCount; + } + +} Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java 2011-01-31 12:13:07 UTC (rev 699) @@ -30,24 +30,18 @@ import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.WikittyService; -import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.entities.BusinessEntity; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyAccessStatHelper; import org.nuiton.wikitty.entities.WikittyAccessStatImpl; import org.nuiton.wikitty.entities.WikittyTokenHelper; -import org.nuiton.wikitty.entities.WikittyTreeNode; -import org.nuiton.wikitty.search.Criteria; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import java.util.Set; /** @@ -223,26 +217,6 @@ } @Override - public WikittyTree restoreTree(String securityToken, String wikittyId) { - WikittyTree result = super.restoreTree(securityToken, wikittyId); - - // before do some work, check if necessary - if (isMonitored(Collections.singleton(WikittyTreeNode.EXT_WIKITTYTREENODE))) { - List<WikittyTreeNode> nodes = new LinkedList<WikittyTreeNode>(); - List<WikittyTree> toVisit = new LinkedList<WikittyTree>(); - toVisit.add(result); - while (!toVisit.isEmpty()) { - WikittyTree tree = toVisit.get(0); - nodes.add(tree.getNode()); - toVisit.addAll(tree.getChildren()); - } - addStat(securityToken, nodes); - } - - return result; - } - - @Override public Wikitty restoreVersion(String securityToken, String wikittyId, String version) { Wikitty result = super.restoreVersion(securityToken, wikittyId, version); addStat(securityToken, Collections.singleton(result)); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceCached.java 2011-01-31 12:13:07 UTC (rev 699) @@ -39,7 +39,6 @@ import org.nuiton.wikitty.entities.WikittyCopyOnWrite; import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.WikittyService; -import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.WikittyUtil; /** @@ -416,18 +415,6 @@ * @return */ @Override - public WikittyTree restoreTree(String securityToken, String wikittyId) { - // TODO lookup in cache, and put in cache - return getDelegate().restoreTree(securityToken, wikittyId); - } - - /** - * just wrap service method - * - * @param wikittyId - * @return - */ - @Override public WikittyEvent deleteTree(String securityToken, String wikittyId) { WikittyEvent result = getDelegate().deleteTree(securityToken, wikittyId); processEvent(result); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceDelegator.java 2011-01-31 12:13:07 UTC (rev 699) @@ -26,26 +26,28 @@ import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.search.PagedResult; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyExtension; -import org.nuiton.wikitty.entities.WikittyTreeNode; import org.nuiton.wikitty.WikittyService; -import org.nuiton.wikitty.WikittyTree; +import org.nuiton.wikitty.search.TreeNodeResult; /** * Wikitty service delegator. - * + * + * Cette classe est abstraite car sa seul utilisation est en en heritant. Cela + * aide pour l'ajout de methode sur WikittyService et voir les sous classes qui + * doivent implanter ou non cette nouvelle methode de facon moins automatique + * qu'une simple delegation + * * @author chatellier * @version $Revision$ * * Last update : $Date$ * By : $Author$ */ -public class WikittyServiceDelegator implements WikittyService { +abstract public class WikittyServiceDelegator implements WikittyService { /** Delegated wikitty service. */ protected WikittyService delegate; @@ -170,8 +172,8 @@ } @Override - public WikittyTree restoreTree(String securityToken, String wikittyId) { - return delegate.restoreTree(securityToken, wikittyId); + public TreeNodeResult<String> findTreeNode(String securityToken, String wikittyId, int depth, boolean count, Criteria filter) { + return delegate.findTreeNode(securityToken, wikittyId, depth, count, filter); } @Override @@ -180,18 +182,6 @@ } @Override - public Entry<String, Integer> restoreNode(String securityToken, - String wikittyId, Criteria filter) { - return delegate.restoreNode(securityToken, wikittyId, filter); - } - - @Override - public Map<String, Integer> restoreChildren(String securityToken, - String wikittyId, Criteria filter) { - return delegate.restoreChildren(securityToken, wikittyId, filter); - } - - @Override public Wikitty restoreVersion(String securityToken, String wikittyId, String version) { return delegate.restoreVersion(securityToken, wikittyId, version); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceNotifier.java 2011-01-31 12:13:07 UTC (rev 699) @@ -39,14 +39,10 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; import org.nuiton.util.ListenerSet; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.PagedResult; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.WikittyService; -import org.nuiton.wikitty.WikittyTree; -import org.nuiton.wikitty.entities.WikittyTreeNode; /** * Wikitty service notifier. Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java 2011-01-31 12:13:07 UTC (rev 699) @@ -32,15 +32,11 @@ import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.PagedResult; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyAuthorisation; import org.nuiton.wikitty.entities.WikittyAuthorisationHelper; @@ -51,7 +47,6 @@ import org.nuiton.wikitty.entities.WikittyMetaExtensionUtil; import org.nuiton.wikitty.WikittyService; import org.nuiton.wikitty.entities.WikittyTokenHelper; -import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.entities.WikittyTreeNode; import org.nuiton.wikitty.entities.WikittyUser; import org.nuiton.wikitty.entities.WikittyUserHelper; @@ -511,22 +506,6 @@ return getDelegate().deleteExtension(securityToken, extNames); } - @Override - public WikittyTree restoreTree(String securityToken, String wikittyId) { - String userId = getUserId(securityToken); - WikittyTree restoredTree = getDelegate().restoreTree(securityToken, wikittyId); - checkRestoreTree(securityToken, userId, restoredTree); - return restoredTree; - } - - protected void checkRestoreTree(String securityToken, String userId, WikittyTree tree) { - checkRestoreTreeNode(securityToken, userId, tree.getNode()); - for (WikittyTree subTree : tree.getChildren()) { - checkRestoreTree(securityToken, userId, subTree); - } - } - - private void checkRestoreTreeNode(String securityToken, String userId, WikittyTreeNode treeNode) { Wikitty treeNodeWikitty = WikittyUtil.getWikitty(getDelegate(), securityToken, treeNode); refuseUnauthorizedRead(securityToken, userId, treeNodeWikitty); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2011-01-31 12:13:07 UTC (rev 699) @@ -31,15 +31,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Date; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.AbstractMap.SimpleEntry; import java.util.LinkedHashSet; -import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,12 +47,11 @@ import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.WikittyService; -import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.entities.WikittyTreeNode; import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; -import org.nuiton.wikitty.entities.WikittyTreeNodeImpl; import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.search.TreeNodeResult; import org.nuiton.wikitty.search.operators.Element; /** @@ -198,7 +194,7 @@ getExtensionStorage().store(tx, allExtensions); WikittyEvent wikUpdate = getWikittyStorage().store(tx, wikitties, force); - getSearchEngine().store(tx, wikitties); + getSearchEngine().store(tx, wikitties, force); WikittyEvent result = new WikittyEvent(this); // prepare update client response @@ -824,57 +820,6 @@ } @Override - public WikittyTree restoreTree(String securityToken, String wikittyId) { - WikittyTransaction tx = WikittyTransaction.get(); - boolean txBeginHere = false; - try { - if (!tx.isStarted()) { - tx.begin(); - txBeginHere = true; - } - - WikittyTree tree = null; - Wikitty w = restore(securityToken, wikittyId); - if (w != null) { - - if (!w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { - throw new WikittyException(String.format( - "Wikitty '%s' do not handle extension %s", - wikittyId, WikittyTreeNode.EXT_WIKITTYTREENODE)); - } - tree = new WikittyTree(); - WikittyTreeNode node = new WikittyTreeNodeImpl(w); - tree.setNode(node); - - WikittyTreeNodeImpl exempleNode = new WikittyTreeNodeImpl(); - exempleNode.setParent(wikittyId); - - Criteria criteria = Search.query(exempleNode.getWikitty()).criteria().setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); - PagedResult<String> childNodesId = findAllByCriteria( - securityToken, criteria); - for (String childNodeId : childNodesId.getAll()) { - tree.addChild(restoreTree(securityToken, childNodeId)); - } - } - - if(txBeginHere) { - tx.commit(); - } - return tree; - } catch (WikittyException ex) { - if (tx != null && tx.isStarted()) { - tx.rollback(); - } - throw ex; - } catch (Exception eee) { - if (tx != null && tx.isStarted()) { - tx.rollback(); - } - throw new WikittyException("Can't restore tree", eee); - } - } - - @Override public WikittyEvent deleteTree(String securityToken, String thesaurusId) { WikittyTransaction tx = WikittyTransaction.get(); boolean txBeginHere = false; @@ -929,8 +874,8 @@ } @Override - public Map.Entry<String, Integer> restoreNode(String securityToken, - String wikittyId, Criteria filter) { + public TreeNodeResult<String> findTreeNode(String securityToken, + String wikittyId, int depth, boolean count, Criteria filter) { WikittyTransaction tx = WikittyTransaction.get(); boolean txBeginHere = false; try { @@ -939,18 +884,11 @@ txBeginHere = true; } - HashMap.SimpleEntry<String, Integer> result = null; + TreeNodeResult<String> result = null; Wikitty w = restore(securityToken, wikittyId); if(w != null) { - if (!w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { - throw new WikittyException(String.format( - "Wikitty '%s' do not handle extension %s", - wikittyId, WikittyTreeNode.EXT_WIKITTYTREENODE)); - } - - Integer count = getSearchEngine().findNodeCount(tx, w, filter); - - result = new SimpleEntry<String, Integer>(wikittyId, count); + result = getSearchEngine().findAllChildrenCount( + tx, wikittyId, depth, count, filter); } if (txBeginHere) { @@ -966,46 +904,6 @@ if (tx != null && tx.isStarted()) { tx.rollback(); } - throw new WikittyException("Can't restore node", eee); - } - } - - @Override - public Map<String, Integer> restoreChildren(String securityToken, - String wikittyId, Criteria filter) { - WikittyTransaction tx = WikittyTransaction.get(); - boolean txBeginHere = false; - try { - if (!tx.isStarted()) { - tx.begin(); - txBeginHere = true; - } - - Map<String, Integer> result = null; - Wikitty w = restore(securityToken, wikittyId); - if(w != null) { - if (!w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { - throw new WikittyException(String.format( - "Wikitty '%s' do not handle extension %s", - wikittyId, WikittyTreeNode.EXT_WIKITTYTREENODE)); - } - - result = getSearchEngine().findAllChildrenCount(tx, w, filter); - } - - if (txBeginHere) { - tx.commit(); - } - return result; - } catch (WikittyException ex) { - if (tx != null && tx.isStarted()) { - tx.rollback(); - } - throw ex; - } catch (Exception eee) { - if (tx != null && tx.isStarted()) { - tx.rollback(); - } throw new WikittyException("Can't restore children", eee); } } @@ -1059,7 +957,7 @@ if(count == numberForCommit) { // Reindex - searchEngine.store(tx, wikitties); + searchEngine.store(tx, wikitties, true); tx.commit(); // Reinit count = 0; @@ -1071,7 +969,7 @@ }); // Last wikitties - searchEngine.store(tx, wikitties); + searchEngine.store(tx, wikitties, true); if (txBeginHere) { tx.commit(); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceTransaction.java 2011-01-31 12:13:07 UTC (rev 699) @@ -28,13 +28,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; @@ -45,9 +43,8 @@ import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.WikittyService; import org.nuiton.wikitty.WikittyServiceFactory; -import org.nuiton.wikitty.WikittyTree; -import org.nuiton.wikitty.entities.WikittyTreeNode; import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.search.TreeNodeResult; /** * @@ -329,27 +326,21 @@ } @Override - public WikittyTree restoreTree(String securityToken, String wikittyId) { + public TreeNodeResult<String> findTreeNode(String securityToken, + String wikittyId, int depth, boolean count, Criteria filter) { + // FIXME throw new UnsupportedOperationException("Not supported yet."); } @Override public WikittyEvent deleteTree(String securityToken, String treeNodeId) { + // FIXME throw new UnsupportedOperationException("Not supported yet."); } @Override - public Entry<String, Integer> restoreNode(String securityToken, String wikittyId, Criteria filter) { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public Map<String, Integer> restoreChildren(String securityToken, String wikittyId, Criteria filter) { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override public Wikitty restoreVersion(String securityToken, String wikittyId, String version) { + // FIXME throw new UnsupportedOperationException("Not supported yet."); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngine.java 2011-01-31 12:13:07 UTC (rev 699) @@ -26,11 +26,11 @@ package org.nuiton.wikitty.storage; import java.util.Collection; -import java.util.Map; import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.search.PagedResult; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.WikittyException; +import org.nuiton.wikitty.search.TreeNodeResult; import org.nuiton.wikitty.services.WikittyTransaction; /** @@ -53,9 +53,11 @@ /** * Store wikitty in storage * Tree are reindexed if necessary. + * @param force if true, force indexation of wikitty, otherwize only dirty + * or new wikitties are indexed */ public void store(WikittyTransaction transaction, - Collection<Wikitty> wikitties); + Collection<Wikitty> wikitties, boolean force); /** * Delete all object with idList argument. If id is not valid or don't exist. @@ -66,27 +68,10 @@ */ public void delete(WikittyTransaction transaction, Collection<String> idList) throws WikittyException; - - // NOTE poussin 20101216 i don't know why this method exists, i suppress it -// /** -// * Delete all object with idList argument. idList is directly passed to search engine -// * and is processed without any other kind of treatment. -// * -// * @param idList list of ids to delete -// * @throws WikittyException -// */ -// public void delete(Collection<String> idList) throws WikittyException; public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria); /** - * Find count of child for a node - * @param w - * @return - */ - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter); - - /** * Find all children ids with attachment count for a node wikitty. * If same attachment found many time in subtree this attachment is count * only once. @@ -112,12 +97,17 @@ * return count for: child1(3), child2(4), child3(8), child4(3), child5(7) * and for the child3 count we have count of subchild1 and subchild2 in * - * but Node and subchild are not returned. This method return only one level - * of tree + * Node and subchild are returned according to depth * - * @param w - * @return + * @param transaction + * @param wikittyId root node to begin + * @param depth depth of node returned, -1 to retrieve all child level + * @param count if true return count of attachment + * @param filter filter on attachment count + * @return Tree start with wikittyId as root + * @throws WikittyException if wikittyId is not WikittyTreeNode */ - public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter); + public TreeNodeResult<String> findAllChildrenCount(WikittyTransaction transaction, + String wikittyId, int depth, boolean count, Criteria filter); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2011-01-31 12:13:07 UTC (rev 699) @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.entities.FieldType; @@ -36,6 +35,7 @@ import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.search.PagedResult; +import org.nuiton.wikitty.search.TreeNodeResult; import org.nuiton.wikitty.search.operators.And; import org.nuiton.wikitty.search.operators.BinaryOperator; import org.nuiton.wikitty.search.operators.Element; @@ -44,7 +44,7 @@ public class WikittySearchEngineInMemory implements WikittySearchEngine { - WikittyStorageInMemory wikittyStorage; + protected WikittyStorageInMemory wikittyStorage; public WikittySearchEngineInMemory(WikittyStorageInMemory wikittyStorage) { this.wikittyStorage = wikittyStorage; @@ -56,7 +56,7 @@ } @Override - public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { + public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties, boolean force) { } @Override @@ -173,12 +173,9 @@ } @Override - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { + public TreeNodeResult<String> findAllChildrenCount(WikittyTransaction transaction, + String wikittyId, int depth, boolean count, Criteria filter) { + // FIXME throw new UnsupportedOperationException("Not supported yet."); } - - @Override - public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - throw new UnsupportedOperationException("Not supported yet."); - } } Modified: trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java =================================================================== --- trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java 2011-01-31 12:13:07 UTC (rev 699) @@ -49,7 +49,6 @@ import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.entities.WikittyLabel; import org.nuiton.wikitty.search.PagedResult; -import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.entities.WikittyTreeNode; import org.nuiton.wikitty.entities.WikittyTreeNodeImpl; import org.nuiton.wikitty.entities.Wikitty; @@ -63,6 +62,7 @@ import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; import org.nuiton.wikitty.search.operators.Element; import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.search.TreeNodeResult; public abstract class StorageTest extends AbstractTestConformance { @@ -505,10 +505,12 @@ assign( table, root, "cat-1" ); assign( bureau, root, "cat-2/subcat-2-4" ); - WikittyTree t = getProxy().restoreTree(root.getWikitty().getId() ); - t.getName().equals("MyCategoryRoot"); + TreeNodeResult<WikittyTreeNode> t = getProxy().findTreeNode( + root.getWikitty().getId(), 0, false, null); + Assert.assertEquals("MyCategoryRoot", t.getObject().getName()); + } protected void assign(Wikitty wikitty, WikittyTreeNodeImpl root, String path) { Added: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/AttachmentInTree.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/AttachmentInTree.java (rev 0) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/AttachmentInTree.java 2011-01-31 12:13:07 UTC (rev 699) @@ -0,0 +1,167 @@ +package org.nuiton.wikitty.storage.solr; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.collections.Factory; +import org.apache.commons.collections.map.LazyMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; +import org.nuiton.wikitty.entities.FieldType.TYPE; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyTreeNode; +import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; + +/** + * Class permettant de construire la liste des objets qui ont ete ajoute + * ou supprimer d'un noeud + * + * @author poussin + * @version $Revision$ + * @since 3.1 + * + * Last update: $Date$ + * by : $Author$ + */ +public class AttachmentInTree { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(AttachmentInTree.class); + + // On genere en meme temps la liste des attachments qui doivent + // etre reindexe + protected Set<String> allAttachmentToIndex = new HashSet<String>(); + protected Factory listFactory = new Factory() { + + @Override + public Object create() { + return new HashSet<String>(); + } + }; + + // key: TreeNode id, value: list of attached id + protected Map<String, Set<String>> attachmentRemovedInTree = LazyMap.decorate( + new HashMap<String, Set<String>>(), listFactory); + // key: TreeNode id, value: list of attached id + protected Map<String, Set<String>> attachmentAddedInTree = LazyMap.decorate( + new HashMap<String, Set<String>>(), listFactory); + + /** + * Remove all ids in attachment list. Ids is object already deleted + * reindex it is not necessary + * + * @param ids + */ + public void clean(Collection<String> ids) { + if (ids != null) { + allAttachmentToIndex.removeAll(ids); + for (Set<String> set : attachmentRemovedInTree.values()) { + set.removeAll(ids); + } + for (Set<String> set : attachmentAddedInTree.values()) { + set.removeAll(ids); + } + } + } + + public int size() { + return allAttachmentToIndex.size(); + } + + public Set<String> getAll() { + return allAttachmentToIndex; + } + + public Map<String, Set<String>> getAdded() { + return attachmentAddedInTree; + } + + public Map<String, Set<String>> getRemoved() { + return attachmentRemovedInTree; + } + + /** + * @param id TreeNode id + * @param ids attachment id + */ + public void remove(String id, Collection<String> attId) { + if (attId != null && !attId.isEmpty()) { + attachmentRemovedInTree.get(id).addAll(attId); + allAttachmentToIndex.addAll(attId); + } + } + + /** + * @param doc TreeNode document representation + */ + public void remove(SolrDocument doc) { + String id = (String)doc.getFieldValue(WikittySolrConstant.SOLR_ID); + + Collection att = doc.getFieldValues(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, + TYPE.WIKITTY)); + remove(id, att); + } + + /** + * @param doc TreeNode document representation + */ + public void remove(SolrInputDocument doc) { + String id = (String)doc.getFieldValue(WikittySolrConstant.SOLR_ID); + + Collection att = doc.getFieldValues(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, + TYPE.WIKITTY)); + remove(id, att); + } + + /** + * @param id TreeNode id + * @param ids attachment id + */ + public void add(String id, Collection<String> attId) { + if (attId != null && !attId.isEmpty()) { + attachmentAddedInTree.get(id).addAll(attId); + allAttachmentToIndex.addAll(attId); + } + } + + /** + * @param doc TreeNode document representation + */ + public void add(SolrDocument doc) { + String id = (String)doc.getFieldValue(WikittySolrConstant.SOLR_ID); + + Collection att = doc.getFieldValues(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, + TYPE.WIKITTY)); + add(id, att); + } + + /** + * @param doc TreeNode document representation + */ + public void add(SolrInputDocument doc) { + String id = (String)doc.getFieldValue(WikittySolrConstant.SOLR_ID); + + Collection att = doc.getFieldValues(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, + TYPE.WIKITTY)); + add(id, att); + } + + /** + * @param doc TreeNode document representation + */ + public void add(Wikitty w) { + String id = w.getId(); + Set<String> att = WikittyTreeNodeHelper.getAttachment(w); + add(id, att); + } + +} Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java 2011-01-31 12:13:07 UTC (rev 699) @@ -382,15 +382,15 @@ } String operand = ""; - StringBuffer result = new StringBuffer(); - result.append(element2solr(contains.getElement())).append(":"); - result.append("("); + StringBuilder result = new StringBuilder(); + result.append(element2solr(contains.getElement())).append(':'); + result.append('('); for (String value : contains.getValue()) { result.append(operand); result.append(value2solr(value)); operand = " OR "; } - result.append(")"); + result.append(')'); return result.toString(); } @@ -418,11 +418,11 @@ } private String isNull2solr(Null isNull) { - return "( *:* - " + WikittySolrConstant.SOLR_NOT_NULL_FIELDS + ":" + isNull.getFieldName() + ")"; + return WikittySolrConstant.SOLR_NULL_FIELD + isNull.getFieldName() + ":true"; } private String isNotNull2solr(Null isNotNull) { - return WikittySolrConstant.SOLR_NOT_NULL_FIELDS + ":" + isNotNull.getFieldName(); + return WikittySolrConstant.SOLR_NULL_FIELD + isNotNull.getFieldName() + ":false"; } private String element2solr(Element element) throws WikittyException { Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrResource.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrResource.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrResource.java 2011-01-31 12:13:07 UTC (rev 699) @@ -58,6 +58,8 @@ static private Log log = LogFactory.getLog(SolrResource.class); protected SolrServer solrServer; + // FIXME poussin 20110131 est-ce vraiment util d'utiliser des ThreadLocal ? + // WikittyTransaction est deja un ThreadLocal protected ThreadLocal<Map<String, SolrInputDocument>> addedDocs; protected ThreadLocal<List<String>> deletedDocs; Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrUtil.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrUtil.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/SolrUtil.java 2011-01-31 12:13:07 UTC (rev 699) @@ -37,10 +37,15 @@ import org.nuiton.wikitty.entities.FieldType.TYPE; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import org.nuiton.wikitty.entities.WikittyTreeNode; import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.SOLR_ID; +import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.TREENODE_PARENTS; +import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.TREENODE_ATTACHED; import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.SUFFIX_BINARY; import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.SUFFIX_BOOLEAN; import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.SUFFIX_DATE; @@ -62,6 +67,24 @@ static private Log log = LogFactory.getLog(SolrUtil.class); /** + * Recherche tous les TreeNode auquel appartient en Attachment l'objet passe + * en parametre + * + * @param doc le document representant l'objet + * @since 3.1 + */ + static public Set<String> getAttachedTreeNode(SolrDocument doc) { + Set<String> result = new HashSet<String>(); + for (String field : doc.getFieldNames()) { + if (field.startsWith(TREENODE_ATTACHED)) { + String id = field.substring(TREENODE_ATTACHED.length()); + result.add(id); + } + } + return result; + } + + /** * Find solr document by id */ static public SolrDocument findById(SolrServer solrServer, String id) { @@ -84,6 +107,80 @@ } /** + * Find solr document by id + * @since 3.1 + */ + static public Map<String, SolrDocument> findAllById( + SolrServer solrServer, Collection<String> ids) { + String solrField = SOLR_ID; + Map<String, SolrDocument> result = findAllByField(solrServer, solrField, ids); + return result; + } + + /** + * Find solr document by TreeNode parents extra field + * @param ids id that must be find in parents list + * @return Map key:TreeNode id, value; solr document associate with id + * @since 3.1 + */ + static public Map<String, SolrDocument> findAllByParents( + SolrServer solrServer, Collection<String> ids) { + String solrField = TREENODE_PARENTS; + Map<String, SolrDocument> result = findAllByField(solrServer, solrField, ids); + return result; + } + + /** + * Find solr document by TreeNode attachment field + * @param ids id that must be find in attachment list + * @return Map key:TreeNode id, value; solr document associate with id + * @since 3.1 + */ + static public Map<String, SolrDocument> findAllByAttachment( + SolrServer solrServer, Collection<String> ids) { + String solrField = SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, TYPE.WIKITTY); + Map<String, SolrDocument> result = findAllByField(solrServer, solrField, ids); + return result; + } + + /** + * Find solr document by specified field in argument + * @param solrServer + * @param solrField field where we looking for ids + * @param ids ids that must be in solrField + * @return Map key:TreeNode id, value; solr document associate with id + * @since 3.1 + */ + static public Map<String, SolrDocument> findAllByField( + SolrServer solrServer, String solrField, Collection<String> ids) { + Map<String, SolrDocument> result = new HashMap<String, SolrDocument>(); + if (ids != null && ids.size() > 0) { + try { + String request = solrField + ":("; + String or = ""; + for (String id : ids) { + id = quoteForSolr(id); + request += or + id; + or = " OR "; + } + request += ")"; + + SolrQuery query = new SolrQuery(request); + QueryResponse response = solrServer.query(query); + SolrDocumentList results = response.getResults(); + for (SolrDocument doc : results) { + String id = (String) doc.getFieldValue(SOLR_ID); + result.put(id, doc); + } + } catch (SolrServerException eee) { + throw new WikittyException("Error during find", eee); + } + } + return result; + } + + /** * if you change this method, change * {@link TypeFieldModifer#convertToField(org.nuiton.wikitty.services.WikittyTransaction, java.lang.String)} * too @@ -112,42 +209,33 @@ } /** - * copy all field of source in new document. - * If include is true copy only field specified in fields - * if include is false copy all field except field in fields. * - * example: - * if doc contains field: abc, aabbcc, aaabbbccc, toto - * copySolrDocument(doc, true, "aa.*", ".*bbb.*") - * field copied are: aabbcc, aaabbbccc - * - * copySolrDocument(doc, false, "aa.*", ".*bbb.*") - * field copied are: abc, toto - * * @param source - * @param include - * @param fields - * @return + * @param dest + * @param fields only copy this field, if null or empty, copy all field */ - static public SolrInputDocument copySolrDocument( - SolrDocument source, boolean include, String... fields) { - SolrInputDocument result = new SolrInputDocument(); + static public void copySolrDocument( + SolrDocument source, SolrInputDocument dest, + String[] fieldToInclude, String[] fieldToExclude) { Collection<String> fieldNames = source.getFieldNames(); Set<String> fieldToCopy = new HashSet<String>(); - if (include) { + if (fieldToInclude == null || fieldToInclude.length == 0) { + fieldToCopy.addAll(fieldNames); + } else { for (String fieldName : fieldNames) { - for (String fieldRegexp : fields) { + for (String fieldRegexp : fieldToInclude) { if (fieldName.matches(fieldRegexp)) { fieldToCopy.add(fieldName); break; } } } - } else { // exclude - fieldToCopy.addAll(fieldNames); + } + + if (fieldToExclude != null && fieldToExclude.length > 0) { for (String fieldName : fieldNames) { - for (String fieldRegexp : fields) { + for (String fieldRegexp : fieldToExclude) { if (fieldName.matches(fieldRegexp)) { fieldToCopy.remove(fieldName); break; @@ -163,15 +251,39 @@ } for (String fieldName : fieldToCopy) { + dest.removeField(fieldName); // to prevent add in already exist dest field Collection<Object> fieldValues = source.getFieldValues(fieldName); for (Object fieldValue : fieldValues) { - result.addField(fieldName, fieldValue); + dest.addField(fieldName, fieldValue); } } - return result; } /** + * + * @param source + * @param dest + * @param fieldToInclude only copy this field, if null or empty, copy all field + * @since 3.1 + */ + static public void copySolrDocument( + SolrDocument source, SolrInputDocument dest, String... fieldToInclude) { + copySolrDocument(source, dest, fieldToInclude, null); + } + + /** + * + * @param source + * @param dest + * @param fields only copy this field, if null or empty, copy all field + * @since 3.1 + */ + static public void copySolrDocumentExcludeSomeField( + SolrDocument source, SolrInputDocument dest, String... fieldToExclude) { + copySolrDocument(source, dest, null, fieldToExclude); + } + + /** * Quote s for solr. Currently only ':' is escaped * @param s to quote * @return new string solr compliant Deleted: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-01-31 12:13:07 UTC (rev 699) @@ -1,773 +0,0 @@ -/* - * #%L - * Wikitty :: wikitty-solr-impl - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2009 - 2010 CodeLutin, Benjamin POUSSIN - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ - -package org.nuiton.wikitty.storage.solr; - -import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.*; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.core.CoreContainer; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.FacetTopic; -import org.nuiton.wikitty.entities.FieldType; -import org.nuiton.wikitty.entities.FieldType.TYPE; -import org.nuiton.wikitty.search.PagedResult; -import org.nuiton.wikitty.entities.WikittyTreeNode; -import org.nuiton.wikitty.entities.Wikitty; -import org.nuiton.wikitty.WikittyException; -import org.nuiton.wikitty.storage.WikittyExtensionStorage; -import org.nuiton.wikitty.storage.WikittySearchEngine; -import org.nuiton.wikitty.services.WikittyTransaction; -import org.nuiton.wikitty.search.Search; - -import java.io.File; -import java.util.Collections; -import org.nuiton.util.ApplicationConfig; -import org.nuiton.wikitty.WikittyConfig; -import org.nuiton.wikitty.WikittyUtil; -import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class WikittySearchEngineSolr implements WikittySearchEngine { - - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(WikittySearchEngineSolr.class); - -// static final public String[] fieldNotToCopyPattern = { -// Pattern.quote(TREENODE_PREFIX) + ".*" -// }; - - /** pattern to copy field from solr document to another solr document - * copy field s_c and s_t are not copied - * tree fields are not copied - * #all. fields are not copied - */ - static final public String[] fieldToCopyPattern = { - // match: id, extensions, not_null_fields - SOLR_ID, SOLR_EXTENSIONS, SOLR_NOT_NULL_FIELDS, - // match: ".*_bi" accepte ce qui fini par _bi - ".*" + SUFFIX_BINARY, - ".*" + SUFFIX_BOOLEAN, - ".*" + SUFFIX_DATE, - ".*" + SUFFIX_NUMERIC, - ".*" + SUFFIX_STRING, - ".*" + SUFFIX_WIKITTY - }; - - /** - * NOTE: On ne pourra utiliser ces patterns pour la copie que lorsque la config - * solr acceptera de creer des copyField avec des expressions regulieres - * Ce qui permettra de genere les champs #all.* via la config solr et non - * pas de devoir les ajouter via le code Java. - * <copyField source="[^.]+\.(.*)" dest="#all.#1"/> - */ - static final public String[] fieldToCopyPatternWithExcludeAll = { - // match: id, extensions, not_null_fields - SOLR_ID, SOLR_EXTENSIONS, SOLR_NOT_NULL_FIELDS, - // match: "(?!(all\.)).*_bi" accept ce qui fini par _bi sauf si ca commence par "all." - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_BINARY, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_BOOLEAN, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_DATE, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_NUMERIC, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_STRING, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_WIKITTY - }; - - /** - * Helper to get information nodes and elements for reindexation. - */ - static protected class ReindexChildTreeNode { - - protected SolrResource solrResource; - protected SolrServer solrServer; - - protected Map<String, Collection<String>> includedNodeIds; - protected Map<String, Collection<String>> excludedNodeIds; - protected Map<String, String> parents; - - public ReindexChildTreeNode(SolrServer solrServer, SolrResource solrResource) { - this.solrServer = solrServer; - this.solrResource = solrResource; - includedNodeIds = new HashMap<String, Collection<String>>(); - excludedNodeIds = new HashMap<String, Collection<String>>(); - parents = new HashMap<String, String>(); - } - - public void putIncludedAttachments(String nodeId, Collection<String> attchmentIds) { - putAttachements(includedNodeIds, nodeId, attchmentIds); - } - - public void putExcludedAttachments(String nodeId, Collection<String> attachmentIds) { - putAttachements(excludedNodeIds, nodeId, attachmentIds); - } - - public void putIncludedAttachment(String nodeId, String attachmentId) { - putAttachment(includedNodeIds, nodeId, attachmentId); - } - - public void putExcludedAttachment(String nodeId, String attachmentId) { - putAttachment(excludedNodeIds, nodeId, attachmentId); - } - - public Collection<String> getExcludedNodeIds(String attachmentId) { - Collection<String> result = excludedNodeIds.get(attachmentId); - if (result == null) { - result = new HashSet<String>(); - } - return result; - } - - public Collection<String> getIncludedNodeIds(String attachmentId) { - Collection<String> result = includedNodeIds.get(attachmentId); - if (result == null) { - result = new HashSet<String>(); - } - return result; - } - - protected void putAttachements(Map<String, Collection<String>> map, String nodeId, Collection<String> attachmentIds) { - if (attachmentIds != null) { - for (String attachmentId : attachmentIds) { - putAttachment(map, nodeId, attachmentId); - } - } - } - - protected void putAttachment(Map<String, Collection<String>> map, String nodeId, String attachmentId) { - Collection<String> values = map.get(attachmentId); - if(values == null) { - values = new HashSet<String>(); - map.put(attachmentId, values); - } - values.add(nodeId); - } - - public void putParent(String nodeId, String parentId) { - parents.put(nodeId, parentId); - } - - public String getParent(String nodeId) { - String parentId = parents.get(nodeId); - - // If not found in map, search in index - if(parentId == null) { - SolrDocument doc = SolrUtil.findById(solrServer, nodeId); - if(doc == null) { - // is root - return null; - } - parentId = (String) doc.getFieldValue(WikittyTreeNode - .FQ_FIELD_WIKITTYTREENODE_PARENT + SUFFIX_WIKITTY); - putParent(nodeId, parentId); - } - - Collection<String> deletedDocIds = solrResource.getDeletedDocs(); - if(deletedDocIds.contains(parentId)) { - return null; - } - return parentId; - } - - public Collection<String> getReindexIds() { - Collection<String> result = new HashSet<String>(); - result.addAll(includedNodeIds.keySet()); - result.addAll(excludedNodeIds.keySet()); - result.addAll(solrResource.getAddedDocIds()); - return result; - } - - /** - * Add in doc fields on association between nodes. - * - * For example if you have a element in node with parent, like this - * A -> B -> C => element, the method add field in document solr : - * TreeNode.root : A - * TreeNode.A : B - * TreeNode.B : C - * TreeNode.C : TreeNode.empty - * - * @throws SolrServerException - */ - public void reindex() throws SolrServerException { - for (String id : getReindexIds()) { - - // Get documents - SolrInputDocument doc = solrResource.getAddedDoc(id); - if(doc == null) { - // Copy old field value - SolrDocument found = SolrUtil.findById(solrServer, id); - if (found != null) { - doc = SolrUtil.copySolrDocument(found, true, fieldToCopyPattern); - solrResource.addDoc(id, doc); - } else { - - if (log.isWarnEnabled()) { - log.warn(String.format("Can't find wikitty id '%s'" - + " in index. Skip this wikitty.", id)); - } - - // If not found just pass - continue; - } - } - - // Add tree node fields - Collection<String> includedChildNodeIds = getIncludedNodeIds(id); - Collection<String> excludedChildNodeIds = getExcludedNodeIds(id); - - // Find all node contain is as attachment - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER - + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT - + SUFFIX_WIKITTY + ":" - + SolrUtil.quoteForSolr(id)); - - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); - iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - - String nodeId = (String) solrDocument.getFieldValue(SOLR_ID); - includedChildNodeIds.add(nodeId); - } - - // Excluded nodes - includedChildNodeIds.removeAll(excludedChildNodeIds); - includedChildNodeIds.removeAll(solrResource.getDeletedDocs()); - - // Add paths in doc - Map<String, String> paths = new HashMap<String, String>(); - for (String nodeId : includedChildNodeIds) { - doc.addField(TREENODE_PREFIX + nodeId, TREENODE_EMPTY); - - // Add path - String childParent = nodeId; - String parent = getParent(childParent); - while (parent != null) { - String parentPath = paths.get(childParent); - if(parentPath == null) { - doc.addField(TREENODE_PREFIX + parent, childParent); - paths.put(childParent, parent); - } - - childParent = parent; - parent = getParent(childParent); - } - - String parentPath = paths.get(childParent); - if(parentPath == null) { - doc.addField(TREENODE_ROOT, childParent); - paths.put(childParent, TREENODE_ROOT); - } - } - } - } - } - - /** solr server */ - protected SolrServer solrServer; - - /** Field modifier use to transform to solr format */ - protected TypeFieldModifier fieldModifier; - - /** JTA resource */ - protected SolrResource solrResource; - - /** - * Init wikitty search engine on solr embedded server. - * - * @param extensionStorage extension storage - * @param properties properties (can be null) - */ - public WikittySearchEngineSolr( - ApplicationConfig config, WikittyExtensionStorage extensionStorage) { - - // init system env solr.data.dir - if (config != null) { - // choix du storage (file or Ram) - String solrDirFactoryKey = - WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_FACTORY.getKey(); - String solrDirFactory = config.getOption(solrDirFactoryKey); - if (solrDirFactory != null) { - System.setProperty(solrDirFactoryKey, solrDirFactory); - } - - // on utilise le directory que si on est pas en Ram - if (solrDirFactory != null && !solrDirFactory.contains("RAMDirectoryFactory")) { - String solrDataDirKey = - WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_DATA.getKey(); - String solrDataDir = config.getOption(solrDataDirKey); - // make sure that dir exists - if (solrDataDir != null) { - File file = new File(solrDataDir); - file.mkdirs(); - System.setProperty(solrDataDirKey, solrDataDir); - } - } - } - - try { - CoreContainer.Initializer initializer = new CoreContainer.Initializer(); - CoreContainer coreContainer = initializer.initialize(); - solrServer = new EmbeddedSolrServer(coreContainer, ""); - - fieldModifier = new TypeFieldModifier(extensionStorage); - solrResource = new SolrResource(solrServer); - - } catch (Exception eee) { - throw new WikittyException("SolR initialization error", eee); - } - } - - @Override - public void clear(WikittyTransaction transaction) { - try { - // FIXME poussin 20100618 pourquoi n'est pas fait dans la transaction ? - solrResource.init(); - solrServer.deleteByQuery("*:*"); - } catch (Exception eee) { - throw new WikittyException("Error during clearing SolR data", eee); - } - } - - @Override - public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { - try { - solrResource.init(); - ReindexChildTreeNode reindexChildTreeNode = - new ReindexChildTreeNode(solrServer, solrResource); - for (Wikitty w : wikitties) { - String id = w.getId(); - - if (w.hasExtension(WikittyTreeNode.EXT_WIKITTYTREENODE)) { - - Set<String> attachments = WikittyTreeNodeHelper.getAttachment(w); - if (attachments == null) { - attachments = Collections.emptySet(); - } - - // - // recherche des anciennes attachments - // - Collection oldAttachments = null; - Set<String> oldAttachmentsCopy; - - // Search deleted attachments - SolrDocument treeNodeDoc = SolrUtil.findById(solrServer, id); - if (treeNodeDoc != null) { - oldAttachments = treeNodeDoc.getFieldValues( - WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + SUFFIX_WIKITTY); - } - // make copy to modify it - if (oldAttachments == null) { - oldAttachments = Collections.emptySet(); - } - oldAttachmentsCopy = new HashSet<String>(oldAttachments); - - - - // les attachments seulement dans old doivent etre exclus - oldAttachmentsCopy.removeAll(attachments); - reindexChildTreeNode.putExcludedAttachments(id, oldAttachmentsCopy); - - if (w.getDirty().contains(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT) - || null == WikittyTreeNodeHelper.getParent(w) - || WikittyUtil.versionGreaterThan("1", w.getVersion())) { - // si le pere a change - // ou qu'il est null (creation d'un nouvel arbre) - // ou que l'objet n'a jamais ete sauve 1 > version - // il faut indexer les attachments - - // il faut reindexer tous les attachments - reindexChildTreeNode.putIncludedAttachments(id, attachments); - - // Get new parent id (may be the same old parent) - String parentId = WikittyTreeNodeHelper.getParent(w); - reindexChildTreeNode.putParent(id, parentId); - } else { - // les attachments seulement dans les nouveaux doivent etre inclus - Set<String> attachmentsCopy = new HashSet<String>(attachments); - attachmentsCopy.removeAll(oldAttachments); - reindexChildTreeNode.putIncludedAttachments(id, attachmentsCopy); - } - - - } - - // Index - SolrInputDocument doc = createIndexDocument(w); - solrResource.addDoc(id, doc); - } - - // Reindex child in tree node - reindexChildTreeNode.reindex(); - } catch (Exception eee) { - throw new WikittyException("Can't store wikitty", eee); - } - } - - @Override - public void delete(WikittyTransaction transaction, Collection<String> ids) throws WikittyException { - try { - solrResource.init(); - ReindexChildTreeNode reindexChildTreeNode = - new ReindexChildTreeNode(solrServer, solrResource); - for (String id : ids) { - - // Find child in node id - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + id + ":*"); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - String childId = (String) solrDocument.getFieldValue(SOLR_ID); - reindexChildTreeNode.putExcludedAttachment(id, childId); - } - - solrResource.deleteDoc(id); - } - - // Reindex child in tree node - reindexChildTreeNode.reindex(); - } catch (Exception eee) { - throw new WikittyException("Can't delete wikitty in index", eee); - } - } - -// @Override -// public void delete(Collection<String> idList) throws WikittyException { -// try { -// for (String id : idList) { -// solrServer.deleteById(id); -// } -// solrServer.commit(); -// } catch (Exception eee) { -// throw new WikittyException("Can't delete wikitty in index", eee); -// } -// } - - @Override - public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { - try { - // Create query with restriction - Restriction2Solr restriction2Solr = new Restriction2Solr(transaction, fieldModifier); - String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); - - // Add paged - int firstIndex = criteria.getFirstIndex(); - int endIndex = criteria.getEndIndex(); - - query.setStart(firstIndex); - int nbRows; - if (endIndex == -1) { - // WARNING It is necessary to substract 'start' otherwise, - // there is a capacity overlow in solR - nbRows = Integer.MAX_VALUE - firstIndex; - } else { - nbRows = endIndex - firstIndex + 1; - } - query.setRows(nbRows); - - // Add sorting - List<String> sortAscending = criteria.getSortAscending(); - if(sortAscending != null) { - for (String sort : sortAscending) { - String tranform = fieldModifier.convertToSolr(transaction, sort); - query.addSortField(tranform, SolrQuery.ORDER.asc); - } - } - - List<String> sortDescending = criteria.getSortDescending(); - if(sortDescending != null) { - for (String sort : sortDescending) { - String tranform = fieldModifier.convertToSolr(transaction, sort); - query.addSortField(tranform, SolrQuery.ORDER.desc); - } - } - - // Add faceting - List<String> facetField = criteria.getFacetField(); - log.debug("facetField : " + facetField); - List<Criteria> facetCriteria = criteria.getFacetCriteria(); - - // use to map query string to criteria facet name - Map<String, String> facetQueryToName = new HashMap<String, String>(); - - if ((facetField != null && !facetField.isEmpty()) - || (facetCriteria != null && !facetCriteria.isEmpty())) { - query.setFacet(true); - query.setFacetMinCount(1); - // query.setFacetLimit(8); // no limit actualy - - // field facetisation - if (facetField != null) { - for (String fqfieldName : facetField) { - String tranform = fieldModifier.convertToSolr(transaction, fqfieldName); - query.addFacetField(tranform); - } - } - - // query facetisation - if (facetCriteria != null) { - for (Criteria facet : facetCriteria) { - String queryFacet = - restriction2Solr.toSolr(facet.getRestriction()); - facetQueryToName.put(queryFacet, facet.getName()); - query.addFacetQuery(queryFacet); - } - } - } - - QueryResponse resp = solrServer.query(query); - SolrDocumentList solrResults = resp.getResults(); - - Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); - if (facetField != null && !facetField.isEmpty()) { - for (FacetField facet : resp.getFacetFields()) { - String facetName = fieldModifier.convertToField(transaction, facet.getName()); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - if (facet.getValues() != null) { - for (FacetField.Count value : facet.getValues()) { - String topicName = value.getName(); - if(!topicName.endsWith(TREENODE_EMPTY)) { - int topicCount = (int) value.getCount(); - FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); - topics.add(topic); - } - } - } - facets.put(facetName, topics); - } - } - if (facetCriteria != null && !facetCriteria.isEmpty()) { - for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { - String facetName = facet.getKey(); - // don't use contains because, map can have key with null value - if (null != facetQueryToName.get(facetName)) { - facetName = facetQueryToName.get(facetName); - } - Integer count = facet.getValue(); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - FacetTopic topic = new FacetTopic(facetName, facetName, count); - topics.add(topic); - facets.put(facetName, topics); - } - } - - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue(SOLR_ID); - ids.add(id); - } - - int numFound = (int)resp.getResults().getNumFound(); - PagedResult<String> result = new PagedResult<String>( - firstIndex, numFound, queryString, facets, ids); - - return result; - } catch (SolrServerException eee) { - throw new WikittyException("Error during find", eee); - } - } - - @Override - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - String wikittyId = w.getId(); - - String parent = WikittyTreeNodeHelper.getParent(w); - if(parent == null) { - parent = TREENODE_ROOT; - } else { - parent = TREENODE_PREFIX + parent; - } - - Criteria criteria = Search.query(filter) - .eq(parent, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(0); - PagedResult<String> search = findAllByCriteria(transaction, criteria); - - int numFound = search.getNumFound(); - return numFound; - } - - - @Override - public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - String wikittyId = w.getId(); - - String parent = WikittyTreeNodeHelper.getParent(w); - if(parent == null) { - parent = TREENODE_ROOT; - } else { - parent = TREENODE_PREFIX + parent; - } - - // Find count with facet, if the node not contain recurcively content, - // the node not found with facet - Criteria criteria = Search.query(filter).eq(parent, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(0) - .addFacetField(TREENODE_PREFIX + wikittyId); - PagedResult<String> search = findAllByCriteria(transaction, criteria); - - Map<String, Integer> counts = new HashMap<String, Integer>(); - List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); - if(topics != null) { - for (FacetTopic topic : topics) { - String topicName = topic.getTopicName(); - int topicCount = topic.getCount(); - counts.put(topicName, topicCount); - } - } - - log.debug("Facet result " + counts); - - // Find all children, add the other node not found with facet - criteria = Search.query().eq(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); - search = findAllByCriteria(transaction, criteria); - - List<String> children = search.getAll(); - for (String child : children) { - if(!counts.containsKey(child)) { - counts.put(child, 0); - } - } - - return counts; - } - - /** - * Create all index document to used to modify indexation. - * this method don't modify index. - * - * The document looks like : - * SolrId : wikittyId - * extensions : extensionNames - * fieldName : fieldValue - * - * @param w all wikitties object to index - * @return solrInputDocument used to modify index - */ - protected SolrInputDocument createIndexDocument(Wikitty w) { - if (log.isDebugEnabled()) { - log.debug("index wikitty " + w.getId()); - } - - SolrInputDocument doc = new SolrInputDocument(); - String id = w.getId(); - doc.addField(SOLR_ID, id); - - for (String name : w.getExtensionNames()) { - doc.addField(SOLR_EXTENSIONS, name); - } - - for (String fqfieldName : w.fieldNames()) { - FieldType fieldType = w.getFieldType(fqfieldName); - TYPE type = fieldType.getType(); - String solrFqFieldName = SolrUtil.getSolrFieldName(fqfieldName, type); - - String solrAllFieldName = SOLR_ALL_EXTENSIONS - + WikittyUtil.FQ_FIELD_NAME_SEPARATOR - + WikittyUtil.getFieldNameFromFQFieldName(solrFqFieldName); - - Object objectValue = w.getFqField(fqfieldName); - if(objectValue != null) { - if (fieldType.isCollection()) { - Collection collectionValue = (Collection) objectValue; - for (Object itemValue : collectionValue) { - if (itemValue != null) { - doc.addField(solrFqFieldName, itemValue); - doc.addField(solrAllFieldName, itemValue); -// -// // Store string field in differents styles -// if(type == TYPE.STRING) { -// doc.addField(solrFqFieldName + SUFFIX_STRING_FULLTEXT, itemValue); -// doc.addField(solrAllFieldName + SUFFIX_STRING_FULLTEXT, itemValue); -// String itemValueLowerCase = itemValue.toString().toLowerCase(); -// doc.addField(solrFqFieldName + SUFFIX_STRING_LOWERCASE, itemValueLowerCase); -// doc.addField(solrAllFieldName + SUFFIX_STRING_LOWERCASE, itemValueLowerCase); -// } -// - doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - if (log.isDebugEnabled()) { - log.debug("index field " + solrFqFieldName + - " with value '" + itemValue + "'"); - } - } - } - } else { - doc.addField(solrFqFieldName, objectValue); - doc.addField(solrAllFieldName, objectValue); -// -// // Store string field in differents styles -// if(type == TYPE.STRING) { -// doc.addField(solrFqFieldName + SUFFIX_STRING_FULLTEXT, objectValue); -// doc.addField(solrAllFieldName + SUFFIX_STRING_FULLTEXT, objectValue); -// String objectValueLowerCase = objectValue.toString().toLowerCase(); -// doc.addField(solrFqFieldName + SUFFIX_STRING_LOWERCASE, objectValueLowerCase); -// doc.addField(solrAllFieldName + SUFFIX_STRING_LOWERCASE, objectValueLowerCase); -// } -// - doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - if (log.isDebugEnabled()) { - log.debug("index field " + solrFqFieldName + - " with value '" + objectValue + "'"); - } - } - } - } - return doc; - } - -} Copied: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java (from rev 681, trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java) =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java (rev 0) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-01-31 12:13:07 UTC (rev 699) @@ -0,0 +1,913 @@ +/* + * #%L + * Wikitty :: wikitty-solr-impl + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2009 - 2010 CodeLutin, Benjamin POUSSIN + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ + +package org.nuiton.wikitty.storage.solr; + +import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.*; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.core.CoreContainer; +import org.nuiton.wikitty.search.Criteria; +import org.nuiton.wikitty.search.FacetTopic; +import org.nuiton.wikitty.entities.FieldType; +import org.nuiton.wikitty.entities.FieldType.TYPE; +import org.nuiton.wikitty.search.PagedResult; +import org.nuiton.wikitty.entities.WikittyTreeNode; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.WikittyException; +import org.nuiton.wikitty.storage.WikittyExtensionStorage; +import org.nuiton.wikitty.storage.WikittySearchEngine; +import org.nuiton.wikitty.services.WikittyTransaction; + +import java.io.File; +import java.util.Collections; +import org.nuiton.util.ApplicationConfig; +import org.nuiton.wikitty.WikittyConfig; +import org.nuiton.wikitty.WikittyUtil; +import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; +import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.search.TreeNodeResult; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittySearchEngineSolr implements WikittySearchEngine { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittySearchEngineSolr.class); + + /** solr server */ + protected SolrServer solrServer; + + /** Field modifier use to transform to solr format */ + protected TypeFieldModifier fieldModifier; + + /** JTA resource */ + protected SolrResource solrResource; + + /** + * Init wikitty search engine on solr embedded server. + * + * @param extensionStorage extension storage + * @param properties properties (can be null) + */ + public WikittySearchEngineSolr( + ApplicationConfig config, WikittyExtensionStorage extensionStorage) { + + // init system env solr.data.dir + if (config != null) { + // choix du storage (file or Ram) + String solrDirFactoryKey = + WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_FACTORY.getKey(); + String solrDirFactory = config.getOption(solrDirFactoryKey); + if (solrDirFactory != null) { + System.setProperty(solrDirFactoryKey, solrDirFactory); + } + + // on utilise le directory que si on est pas en Ram + if (solrDirFactory != null && !solrDirFactory.contains("RAMDirectoryFactory")) { + String solrDataDirKey = + WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_DATA.getKey(); + String solrDataDir = config.getOption(solrDataDirKey); + // make sure that dir exists + if (solrDataDir != null) { + File file = new File(solrDataDir); + if (!file.exists() && !file.mkdirs()) { + throw new WikittyException(String.format( + "Can't create directory '%s'", solrDataDir)); + } + log.info(String.format("Use SolR directory '%s'", solrDataDir)); + System.setProperty(solrDataDirKey, solrDataDir); + } + } + } + + try { + CoreContainer.Initializer initializer = new CoreContainer.Initializer(); + CoreContainer coreContainer = initializer.initialize(); + solrServer = new EmbeddedSolrServer(coreContainer, ""); + + fieldModifier = new TypeFieldModifier(extensionStorage); + solrResource = new SolrResource(solrServer); + + } catch (Exception eee) { + throw new WikittyException("SolR initialization error", eee); + } + } + + @Override + public void clear(WikittyTransaction transaction) { + try { + // FIXME poussin 20100618 pourquoi n'est pas fait dans la transaction ? + solrResource.init(); + solrServer.deleteByQuery("*:*"); + } catch (Exception eee) { + throw new WikittyException("Error during clearing SolR data", eee); + } + } + + @Override + public void store(WikittyTransaction transaction, + Collection<Wikitty> wikitties, boolean force) { + try { + solrResource.init(); + + // tous les wikitties passes en parametre + Map<String, Wikitty> allWikitties = new HashMap<String, Wikitty>(); + // les ids des wikitties en parametre reellement modifier (a reindexer) + Set<String> dirtyObject = new HashSet<String>(); + // les ids des TreeNodes dont le champs parent a change (est aussi + // contenu dans dirtyObject + Set<String> dirtyParent = new HashSet<String>(); + // les valeur du champs parent des TreeNodes dont le champs parent + // a change (sauf si parent = null) + Set<String> dirtyParentParentId = new HashSet<String>(); + + // remplissage des collections + for(Wikitty w : wikitties) { + allWikitties.put(w.getId(), w); + if (force || !w.getDirty().isEmpty() || + WikittyUtil.versionGreaterThan("1", w.getVersion())) { + // s'il y a au moins un champs a reindexer ou que l'objet + // n'a jamais ete sauve (1 > version) + dirtyObject.add(w.getId()); + if (WikittyTreeNodeHelper.hasExtension(w) && + (w.getDirty().contains(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT) + || null == WikittyTreeNodeHelper.getParent(w))) { + // si le pere a change + // ou qu'il est null (creation d'un nouvel arbre) + // il faut indexer le noeud + dirtyParent.add(w.getId()); + String parent = WikittyTreeNodeHelper.getParent(w); + if (parent != null) { + dirtyParentParentId.add(parent); + } + } + } + } + + // recuperation des documents Solr deja indexes, pour minimiser la reindexation + Map<String, SolrDocument> dirtyObjectDoc = + SolrUtil.findAllById(solrServer, dirtyObject); + Map<String, SolrDocument> dirtyParentDoc = + SolrUtil.findAllByParents(solrServer, dirtyParent); + Map<String, SolrDocument> parents = + SolrUtil.findAllById(solrServer, dirtyParentParentId); + + // On genere en meme temps la liste des attachments qui doivent + // etre reindexe + AttachmentInTree attachmentInTree = new AttachmentInTree(); + + // + // Phase 1: on indexe les objets passe en paremetre, on copie si + // besoin #tree.attached et #tree.* des TreeNode dont + // leur champs parent n'a pas ete modifie, et dans ce cas + // on collecte les modif d'attachments des TreeNode + // + + for(String id : dirtyObject) { + Wikitty w = allWikitties.get(id); + SolrDocument oldDoc = dirtyObjectDoc.get(id); + SolrInputDocument doc = createIndexDocument(w); + if (oldDoc != null) { + // copy des champs #tree.attached + SolrUtil.copySolrDocument(oldDoc, doc, TREENODE_ATTACHED + ".*"); + if (WikittyTreeNodeHelper.hasExtension(w) + && !dirtyParentDoc.containsKey(id)) { + // si c'est un TreeNode, mais qu'aucun pere n'a change + // on recopie l'ancienne indexation d'arbre + // si elle existe + SolrUtil.copySolrDocument(oldDoc, doc, TREENODE_PREFIX + ".*"); + + // il faut verifier les objets attaches + // attaches ajoute/supprime + // on ne traite ici que les TreeNode sans modif d'indexation + // pour les autres les attachments seront traites dans + // la phase suivante + Set<String> newAtt = WikittyTreeNodeHelper.getAttachment(w); + Collection oldAtt = oldDoc.getFieldValues(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, + TYPE.WIKITTY)); + // il faut supprimer l'indexation arbre des noeuds + // qui sont dans old, mais pas dans new + Set<String> toRemove = new HashSet<String>(); + if (oldAtt != null) { + toRemove.addAll(oldAtt); + } + if (newAtt != null) { + toRemove.removeAll(newAtt); + } + attachmentInTree.remove(id, toRemove); + // il faut ajouter l'indexation arbre des noeuds + // qui sont dans new, mais pas dans old + Set<String> toAdd = new HashSet<String>(); + if (newAtt != null) { + toAdd.addAll(newAtt); + } + if (oldAtt != null) { + toAdd.removeAll(oldAtt); + } + attachmentInTree.add(id, toAdd); + } + } + solrResource.addDoc(id, doc); + } + + // + // Phase 2: on reindexe tous les TreeNode qui en ont besoin + // nouveau TreeNode ou TreeNode ayant un parent modifie + // + + // on ajoute tous les TreeNode qui doivent aussi etre reindexe + // noeud du sous arbre d'un noeud dont le pere a ete modifie + dirtyParent.addAll(dirtyParentDoc.keySet()); + + for (String id : dirtyParent) { + // w et oldDoc peuvent etre null, mais pas en meme temps + // w est null si c'est un noeud dont la reindexation est force + // parce que un de ces peres a change de parent + // oldDoc est null, si l'objet n'a jamais ete indexe (nouveau) + Wikitty w = allWikitties.get(id); + SolrDocument oldDoc = dirtyParentDoc.get(id); + SolrInputDocument doc = solrResource.getAddedDoc(id); + if (w == null) { + // on reindexe un ancien objet + // normalement doc doit etre null + doc = new SolrInputDocument(); + // on recopie tous les champs, sauf l'indexation arbre + SolrUtil.copySolrDocumentExcludeSomeField( + oldDoc, doc, TREENODE_PREFIX + ".*"); + + // modifie les champs root, parents + addTreeIndexField(solrResource, doc, parents); + + attachmentInTree.remove(oldDoc); + attachmentInTree.add(oldDoc); + } else if (oldDoc == null) { + // ajoute les champs root, parents + addTreeIndexField(solrResource, doc, parents); + + // on indexe un nouvel objet, il faut ajouter tous les + // attachment pour indexation + attachmentInTree.add(w); + } else { + // ni w, ni oldDoc ne sont pas null, c'est une modification + // dans la phase precendente on a deja indexe les champs + // normaux + + // ajoute les champs root, parents + addTreeIndexField(solrResource, doc, parents); + + // il faut supprimer tous les anciens attaches + // et ajouter tous nouveaux pour la reindexation + attachmentInTree.remove(oldDoc); + attachmentInTree.add(w); + } + solrResource.addDoc(id, doc); + } + + // + // Phase 3: on reindexe les attachments qui en ont besoin + // + + // on passe null pour tree, car tous les noeuds doivent etre dans + // solrResource + addTreeIndexField(solrResource, null, attachmentInTree); + } catch (Exception eee) { + throw new WikittyException("Can't store wikitty", eee); + } + } + + /** + * Plusieurs actions possibles en fontion du type d'objet: + * + * <li> suppression d'un objet NON noeud + * <li> suppression de cet objets + * <li> suppression de cet objets dans les attachments des noeuds qui le contiennent + * </li> + * <li> suppression d'un noeud d'arbre + * <li> suppression du noeud + * <li> reindexation des noeuds qui le contenait comme parent + * <li> suppression des attached sur les objets contenus dans les attachments de ce noeud + * <li> reindexation des objets qui le contenait comme parent dans un champs attached + * </li> + * + * @param transaction + * @param ids + * @throws WikittyException + */ + @Override + public void delete(WikittyTransaction transaction, Collection<String> ids) throws WikittyException { + try { + solrResource.init(); + + AttachmentInTree attachmentInTree = new AttachmentInTree(); + + // list de tous les objets a supprimer + Map<String, SolrDocument> removed = + SolrUtil.findAllById(solrServer, ids); + // list des TreeNode supprimer + Map<String, SolrDocument> treeNodeRemoved = + new HashMap<String, SolrDocument>(); + + // list des TreeNode dont des attachments ont ete supprime + Set<String> treeNodeAttachmentRemovedId = new HashSet<String>(); + + // + // Phase 1: Suppression des objets demandes en parametre + // + for (Map.Entry<String, SolrDocument> e : removed.entrySet()) { + String id = e.getKey(); + SolrDocument doc = e.getValue(); + if (doc.containsKey(TREENODE_ROOT)) { + treeNodeRemoved.put(id, doc); + attachmentInTree.remove(doc); + } + // recherche tous les noeuds sur lequel est attache cet objet + Set<String> treeNodeIds = SolrUtil.getAttachedTreeNode(doc); + treeNodeAttachmentRemovedId.addAll(treeNodeIds); + + solrResource.deleteDoc(id); + } + + // + // Phase 2: Suppression dans les listes d'attachment des TreeNode supprimes + // + + // on essaie pas de modifier un noeud que l'on vient de supprimer + treeNodeAttachmentRemovedId.removeAll(ids); + if (treeNodeAttachmentRemovedId.size() > 0) { + + // recuperation des noeuds dont il faut modifier les attachments + Map<String, SolrDocument> treeNodeAttachmentRemoved = + SolrUtil.findAllById(solrServer, treeNodeAttachmentRemovedId); + for (Map.Entry<String, SolrDocument> e : treeNodeAttachmentRemoved.entrySet()) { + String id = e.getKey(); + + // le noeud n'a pas ete supprime, donc il faut le mettre a jour + SolrDocument doc = e.getValue(); + SolrInputDocument newDoc = new SolrInputDocument(); + String field = SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT, + TYPE.WIKITTY); + SolrUtil.copySolrDocumentExcludeSomeField(doc, newDoc, field); + Collection atts = doc.getFieldValues(field); + // remove deleted attachment + Set<String> newAtts = new HashSet<String>(atts); + newAtts.removeAll(ids); + // si apres effacement des attachments, il n'y a plus + // d'attachment sur le node, cela revient a avoir + // le field attachment null + if (newAtts.isEmpty()) { + newAtts = null; + } + + addToIndexDocument(newDoc, TYPE.WIKITTY, field, newAtts); + solrResource.addDoc(id, newDoc); + } + } + + // les phases suivantes ne sont a faire que s'il y a des TreeNode + // supprimes + Map<String, SolrDocument> treeNodeParentRemoved = Collections.emptyMap(); + if (treeNodeRemoved != null && treeNodeRemoved.size() > 0) { + // + // Phase 3: reindexation des noeuds dont un pere a disparu + // + + // list des TreeNode dont un des pere a ete supprime + treeNodeParentRemoved = + SolrUtil.findAllByParents(solrServer, ids); + + // on commence par supprimer des noeuds a reindexer les noeuds supprimes + treeNodeParentRemoved.keySet().removeAll(treeNodeRemoved.keySet()); + + for (Map.Entry<String, SolrDocument> e : treeNodeParentRemoved.entrySet()) { + SolrDocument oldDoc = e.getValue(); + + SolrInputDocument doc = new SolrInputDocument(); + // on recopie tous les champs, sauf l'indexation arbre + SolrUtil.copySolrDocumentExcludeSomeField( + oldDoc, doc, TREENODE_PREFIX + ".*"); + + // modifie les champs root, parents + addTreeIndexField(solrResource, doc, treeNodeParentRemoved); + + // il faut reindexer tous les attachments de ce noeud + attachmentInTree.remove(oldDoc); + attachmentInTree.add(doc); + } + + // + // Phase 4: on reindexe les attachments qui en ont besoin + // + attachmentInTree.clean(ids); + addTreeIndexField(solrResource, treeNodeParentRemoved, attachmentInTree); + } + } catch (Exception eee) { + throw new WikittyException("Can't delete wikitty in index", eee); + } + } + + /** + * Modifie/Ajoute les champs specifique a l'indexation des arbres sur les + * TreeNode. + * + * On se base sur le fait que si un TreeNode est dans SolrResource il ne + * peut etre que dans deux etats. Soit il a ete reindexe pour les arbres + * et il a les champs d'indexation arbre. Soit il a pas encore ete reindexe + * pour les arbres et dans ce cas il ne doit pas avoir les champs d'indexation + * d'arbre. (il est donc interdit d'avoir des champs d'indexation arbre + * obsolete si le document est dans SolrResource) + * + * @param doc les documents representant le TreeNode + * @param tree tous les autres noeuds d'arbre dont on pourrait avoir + * besoin pour l'indexation + */ + protected void addTreeIndexField(SolrResource solrResource, + SolrInputDocument doc, Map<String, SolrDocument> tree) { + Set<String> parents = new HashSet<String>(); + String root = null; + String parentId = (String)doc.getFieldValue(SOLR_ID); + parents.add(parentId); + while (root == null) { + String nextParentId = null; + SolrInputDocument parentDoc = solrResource.getAddedDoc(parentId); + if (parentDoc != null) { + if (parentDoc != null) { + // si parentDoc a deja ete indexe pour l'arbre, on peut reutiliser + // directement les valeurs et sortir de la boucle + if (parentDoc.containsKey(TREENODE_ROOT)) { + root = (String) parentDoc.getFieldValue(TREENODE_ROOT); + Collection p = parentDoc.getFieldValues(TREENODE_PARENTS); + parents.addAll(p); + break; + } else { + nextParentId = (String) parentDoc.getFieldValue(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, TYPE.WIKITTY)); + } + } + } else { + SolrDocument oldParentDoc = tree.get(parentId); + if (oldParentDoc != null) { + // si parentDoc a deja ete indexe pour l'arbre, on peut reutiliser + // directement les valeurs et sortir de la boucle + if (oldParentDoc.containsKey(TREENODE_ROOT)) { + root = (String) oldParentDoc.getFieldValue(TREENODE_ROOT); + Collection p = oldParentDoc.getFieldValues(TREENODE_PARENTS); + parents.addAll(p); + break; + } else { + nextParentId = (String) oldParentDoc.getFieldValue(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, TYPE.WIKITTY)); + } + } + } + if (nextParentId != null) { + parents.add(nextParentId); + parentId = nextParentId; + } else { + root = parentId; + } + } + + doc.removeField(TREENODE_ROOT); + doc.removeField(TREENODE_PARENTS); + doc.removeField(TREENODE_DEPTH); + + doc.addField(TREENODE_ROOT, root); + doc.addField(TREENODE_DEPTH, parents.size()); + for (String id : parents) { + doc.addField(TREENODE_PARENTS, id); + } + } + + /** + * Update attaced extra field on all objects passed in argument allAttachmentToIndex + * + * @param solrResource must contains reindexed TreeNode, that contains attachment + * @param allAttachmentToIndex id of object to update + * @param attachmentRemovedInTree index to remove + * @param attachmentAddedInTree index to add + */ + protected void addTreeIndexField(SolrResource solrResource, + Map<String, SolrDocument> tree, AttachmentInTree attachmentInTree) { + + if (attachmentInTree.size() > 0) { + Map<String, SolrDocument> attachments = + SolrUtil.findAllById(solrServer, attachmentInTree.getAll()); + + for (String treeNodeId : attachmentInTree.getRemoved().keySet()) { + for (String attId : attachmentInTree.getRemoved().get(treeNodeId)) { + SolrDocument oldDoc = attachments.get(attId); + SolrInputDocument doc = solrResource.getAddedDoc(attId); + if (doc == null) { + doc = new SolrInputDocument(); + SolrUtil.copySolrDocument(oldDoc, doc); + solrResource.addDoc(attId, doc); + } + doc.remove(TREENODE_ATTACHED + treeNodeId); + } + } + for (String treeNodeId : attachmentInTree.getAdded().keySet()) { + Collection treeNodeParents = null; + SolrInputDocument treeNodeDoc = solrResource.getAddedDoc(treeNodeId); + if (treeNodeDoc != null) { + treeNodeParents = treeNodeDoc.getFieldValues(TREENODE_PARENTS); + } else if (tree != null) { + SolrDocument doc = tree.get(treeNodeId); + treeNodeParents = doc.getFieldValues(TREENODE_PARENTS); + } else { + log.error("SolR doc not found in Transaction or in tree." + + "This is a bug !!!"); + } + for (String attId : attachmentInTree.getAdded().get(treeNodeId)) { + SolrDocument oldDoc = attachments.get(attId); + SolrInputDocument doc = solrResource.getAddedDoc(attId); + if (doc == null) { + doc = new SolrInputDocument(); + SolrUtil.copySolrDocument(oldDoc, doc); + solrResource.addDoc(attId, doc); + } + doc.removeField(TREENODE_ATTACHED + treeNodeId); + for (Object id : treeNodeParents) { + doc.addField(TREENODE_ATTACHED + treeNodeId, id); + } + } + } + } + } + + @Override + public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { + try { + // Create query with restriction + Restriction2Solr restriction2Solr = new Restriction2Solr(transaction, fieldModifier); + String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); + + // Add paged + int firstIndex = criteria.getFirstIndex(); + int endIndex = criteria.getEndIndex(); + + query.setStart(firstIndex); + int nbRows; + if (endIndex == -1) { + // WARNING It is necessary to substract 'start' otherwise, + // there is a capacity overlow in solR + nbRows = Integer.MAX_VALUE - firstIndex; + } else { + nbRows = endIndex - firstIndex + 1; + } + query.setRows(nbRows); + + // Add sorting + List<String> sortAscending = criteria.getSortAscending(); + if(sortAscending != null) { + for (String sort : sortAscending) { + String tranform = fieldModifier.convertToSolr(transaction, sort); + query.addSortField(tranform, SolrQuery.ORDER.asc); + } + } + + List<String> sortDescending = criteria.getSortDescending(); + if(sortDescending != null) { + for (String sort : sortDescending) { + String tranform = fieldModifier.convertToSolr(transaction, sort); + query.addSortField(tranform, SolrQuery.ORDER.desc); + } + } + + // Add faceting + List<String> facetField = criteria.getFacetField(); + log.debug("facetField : " + facetField); + List<Criteria> facetCriteria = criteria.getFacetCriteria(); + + // use to map query string to criteria facet name + Map<String, String> facetQueryToName = new HashMap<String, String>(); + + if ((facetField != null && !facetField.isEmpty()) + || (facetCriteria != null && !facetCriteria.isEmpty())) { + query.setFacet(true); + query.setFacetMinCount(1); + // query.setFacetLimit(8); // no limit actualy + + // field facetisation + if (facetField != null) { + for (String fqfieldName : facetField) { + String tranform = fieldModifier.convertToSolr(transaction, fqfieldName); + query.addFacetField(tranform); + } + } + + // query facetisation + if (facetCriteria != null) { + for (Criteria facet : facetCriteria) { + String queryFacet = + restriction2Solr.toSolr(facet.getRestriction()); + facetQueryToName.put(queryFacet, facet.getName()); + query.addFacetQuery(queryFacet); + } + } + } + + if(log.isDebugEnabled()) { + log.debug(String.format("Try to execute query %s", query)); + } + QueryResponse resp = solrServer.query(query); + SolrDocumentList solrResults = resp.getResults(); + + Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); + if (facetField != null && !facetField.isEmpty()) { + for (FacetField facet : resp.getFacetFields()) { + String facetName = fieldModifier.convertToField(transaction, facet.getName()); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + if (facet.getValues() != null) { + for (FacetField.Count value : facet.getValues()) { + String topicName = value.getName(); +// if(!topicName.endsWith(TREENODE_EMPTY)) { + int topicCount = (int) value.getCount(); + FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); + topics.add(topic); +// } + } + } + facets.put(facetName, topics); + } + } + if (facetCriteria != null && !facetCriteria.isEmpty()) { + for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { + String facetName = facet.getKey(); + // don't use contains because, map can have key with null value + if (null != facetQueryToName.get(facetName)) { + facetName = facetQueryToName.get(facetName); + } + Integer count = facet.getValue(); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + FacetTopic topic = new FacetTopic(facetName, facetName, count); + topics.add(topic); + facets.put(facetName, topics); + } + } + + List<String> ids = new ArrayList<String>(solrResults.size()); + for (SolrDocument doc : solrResults) { + String id = (String) doc.getFieldValue(SOLR_ID); + ids.add(id); + } + + int numFound = (int)resp.getResults().getNumFound(); + PagedResult<String> result = new PagedResult<String>( + firstIndex, numFound, queryString, facets, ids); + + return result; + } catch (SolrServerException eee) { + throw new WikittyException("Error during find", eee); + } + } + + /** + * Si l'argument n'est pas un TreeNode, une exception est levee + * + * @param transaction + * @param w l'objet root du resultat + * @param depth profondeur souhaite pour la recherche des fils + * @param count vrai si l'on souhaite avoir le nombre d'attachment associe + * au noeud retourne + * @param filter filtre utilise pour compter le nombre d'attachment + * @return + */ + @Override + public TreeNodeResult<String> findAllChildrenCount( + WikittyTransaction transaction, + String wikittyId, int depth, boolean count, Criteria filter) { + try { + TreeNodeResult<String> result = null; + + SolrDocument doc = SolrUtil.findById(solrServer, wikittyId); + if (doc != null) { + // on verifie que l'argument est bien un TreeNode + if (doc.containsKey(TREENODE_DEPTH)) { + + Search treeSearch = Search.query().and().eq(TREENODE_PARENTS, wikittyId); + if (depth >= 0) { + Integer d = (Integer) doc.getFieldValue(TREENODE_DEPTH); + int startDepth = d.intValue(); + treeSearch = treeSearch.bw(TREENODE_DEPTH, + String.valueOf(startDepth), + String.valueOf(startDepth + depth)); + } + Criteria treeCriteria = treeSearch.criteria(); + + // on a dans treeSearch uniquement le noeud passe en parametre + // et ses enfants jusqu'a la profondeur demandee + Restriction2Solr restriction2Solr = + new Restriction2Solr(transaction, fieldModifier); + String queryString = restriction2Solr.toSolr( + treeCriteria.getRestriction(), solrServer); + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); + QueryResponse resp = solrServer.query(query); + SolrDocumentList solrResults = resp.getResults(); + + Map<String, Integer> counts = new HashMap<String, Integer>(); + // recuperation si demande du nombre d'attachment par noeud + if (count) { + // TODO poussin 20110128 regarder si on ne peut pas + // restreindre les facettes au noeud trouve dans la recherche + // precedente + Criteria attCriteria = Search.query(filter).eq( + TREENODE_ATTACHED_ALL, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(0) + .addFacetField(TREENODE_ATTACHED_ALL); + PagedResult<String> attSearch = + findAllByCriteria(transaction, attCriteria); + List<FacetTopic> topics = attSearch.getTopic(TREENODE_ATTACHED_ALL); + if (topics != null) { + for (FacetTopic topic : topics) { + String topicName = topic.getTopicName(); + int topicCount = topic.getCount(); + counts.put(topicName, topicCount); + } + } + } + + // construction du resultat, il proceder en 2 phases car + // sinon si on construit un fils avant son pere, il ne sera + // jamais associe + Map<String, TreeNodeResult<String>> allTreeNodeResult = + new HashMap<String, TreeNodeResult<String>>(); + // key: id de l'enfant, value: l'id du parent + Map<String, String> childParent = new HashMap<String, String>(); + // construction de tous les TreeNodeResult qui permettront + // de construire l'arbre + for (SolrDocument d : solrResults) { + String id = (String) d.getFieldValue(SOLR_ID); + String parentId = (String) d.getFieldValue(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, + TYPE.WIKITTY)); + int nb = counts.containsKey(id) ? counts.get(id) : 0; + TreeNodeResult<String> child = new TreeNodeResult<String>(id, nb); + allTreeNodeResult.put(id, child); + childParent.put(id, parentId); + } + // construction de l'arbre avant de le retourner + for(Map.Entry<String, TreeNodeResult<String>> e : allTreeNodeResult.entrySet()) { + String id = e.getKey(); + String parentId = childParent.get(id); + if (allTreeNodeResult.containsKey(parentId)) { + TreeNodeResult<String> child = e.getValue(); + TreeNodeResult<String> parent = allTreeNodeResult.get(parentId); + parent.add(child); + } + } + result = allTreeNodeResult.get(wikittyId); + } else { + throw new WikittyException(String.format( + "Wikitty '%s' do not handle extension %s", + wikittyId, WikittyTreeNode.EXT_WIKITTYTREENODE)); + } + } + return result; + } catch (SolrServerException eee) { + throw new WikittyException("Error during find", eee); + } + + } + + protected void addToIndexDocument(SolrInputDocument doc, + TYPE type, String fqfieldName, Object fieldValue) { + if (fqfieldName.startsWith(SOLR_WIKITTY_PREFIX)) { + doc.remove(fqfieldName); + doc.addField(fqfieldName, fieldValue); + } else { + String solrFqFieldName = SolrUtil.getSolrFieldName(fqfieldName, type); + + // #all.<fieldname> + // permet de faire des recherches inter extension sur un champs ayant + // le meme nom. ex:Person.name et User.name + String solrAllFieldName = SOLR_ALL_EXTENSIONS + + WikittyUtil.FQ_FIELD_NAME_SEPARATOR + + WikittyUtil.getFieldNameFromFQFieldName(solrFqFieldName); + + String solrNullFieldFqFieldName = SOLR_NULL_FIELD + fqfieldName; + + doc.remove(solrFqFieldName); + doc.remove(solrNullFieldFqFieldName); + doc.remove(solrAllFieldName); + + String solrNullFieldFqFieldNameValue = "true"; + if(fieldValue != null) { + doc.addField(solrFqFieldName, fieldValue); + doc.addField(solrAllFieldName, fieldValue); + solrNullFieldFqFieldNameValue = "false"; + if (log.isDebugEnabled()) { + log.debug("index field " + solrFqFieldName + + " with value '" + fieldValue + "'"); + } + } + doc.addField(solrNullFieldFqFieldName, solrNullFieldFqFieldNameValue); + } + } + + /** + * modify one field in SolrInputDocument + * + * @param doc SolrInputDocument to modify + * @param w wikitty used to find field value + * @param fqfieldName field to index + */ + protected void addToIndexDocument(SolrInputDocument doc, Wikitty w, String fqfieldName) { + TYPE type = null; + Object fieldValue; + if (SOLR_ID.equals(fqfieldName)) { + // extra field #id + fieldValue = w.getId(); + } else if (SOLR_EXTENSIONS.equals(fqfieldName)) { + // extra field #extensions + fieldValue= w.getExtensionNames(); + } else { + // un champs normal + FieldType fieldType = w.getFieldType(fqfieldName); + type = fieldType.getType(); + + fieldValue = w.getFqField(fqfieldName); + + } + addToIndexDocument(doc, type, fqfieldName, fieldValue); + } + + /** + * Create all index document to used to modify indexation. + * this method don't modify index. + * + * The document looks like : + * #id : wikittyId + * #extensions : extensionNames + * [fieldName] : [fieldValue] + * #null_field-[fieldname] : [true|false] + * + * @param w all wikitties object to index + * @return solrInputDocument used to modify index + */ + protected SolrInputDocument createIndexDocument(Wikitty w) { + if (log.isDebugEnabled()) { + log.debug("index wikitty " + w.getId()); + } + + SolrInputDocument doc = new SolrInputDocument(); + addToIndexDocument(doc, w, SOLR_ID); + addToIndexDocument(doc, w, SOLR_EXTENSIONS); + // iter sur tous les champs et pas seulement sont qui ont une valeur + // pour pouvoir les indexer comme des champs a null + for (String fqfieldName : w.getAllFieldNames()) { + addToIndexDocument(doc, w, fqfieldName); + } + return doc; + } + +} Deleted: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java 2011-01-31 12:13:07 UTC (rev 699) @@ -1,716 +0,0 @@ -/* - * #%L - * Wikitty :: wikitty-solr-impl - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2009 - 2010 CodeLutin, Benjamin POUSSIN - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ - -package org.nuiton.wikitty.storage.solr; - -import static org.nuiton.wikitty.storage.solr.WikittySolrConstant.*; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.core.CoreContainer; -import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.FacetTopic; -import org.nuiton.wikitty.entities.FieldType; -import org.nuiton.wikitty.entities.FieldType.TYPE; -import org.nuiton.wikitty.search.PagedResult; -import org.nuiton.wikitty.entities.WikittyTreeNode; -import org.nuiton.wikitty.entities.Wikitty; -import org.nuiton.wikitty.WikittyException; -import org.nuiton.wikitty.storage.WikittyExtensionStorage; -import org.nuiton.wikitty.storage.WikittySearchEngine; -import org.nuiton.wikitty.services.WikittyTransaction; - -import java.io.File; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import org.nuiton.util.ApplicationConfig; -import org.nuiton.wikitty.WikittyConfig; -import org.nuiton.wikitty.WikittyUtil; -import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; -import org.nuiton.wikitty.search.Search; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class WikittySearchEngineSolrIndexInTreeNode implements WikittySearchEngine { - - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(WikittySearchEngineSolr.class); - -// static final public String[] fieldNotToCopyPattern = { -// Pattern.quote(TREENODE_PREFIX) + ".*" -// }; - - /** pattern to copy field from solr document to another solr document - * copy field s_c and s_t are not copied - * tree fields are not copied - * #all. fields are not copied - */ - static final public String[] fieldToCopyPattern = { - // match: id, extensions, not_null_fields - SOLR_ID, SOLR_EXTENSIONS, SOLR_NOT_NULL_FIELDS, - // match: ".*_bi" accepte ce qui fini par _bi - ".*" + SUFFIX_BINARY, - ".*" + SUFFIX_BOOLEAN, - ".*" + SUFFIX_DATE, - ".*" + SUFFIX_NUMERIC, - ".*" + SUFFIX_STRING, - ".*" + SUFFIX_WIKITTY - }; - - /** - * NOTE: On ne pourra utiliser ces patterns pour la copie que lorsque la config - * solr acceptera de creer des copyField avec des expressions regulieres - * Ce qui permettra de genere les champs #all.* via la config solr et non - * pas de devoir les ajouter via le code Java. - * <copyField source="[^.]+\.(.*)" dest="#all.#1"/> - */ - static final public String[] fieldToCopyPatternWithExcludeAll = { - // match: id, extensions, not_null_fields - SOLR_ID, SOLR_EXTENSIONS, SOLR_NOT_NULL_FIELDS, - // match: "(?!(all\.)).*_bi" accept ce qui fini par _bi sauf si ca commence par "all." - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_BINARY, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_BOOLEAN, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_DATE, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_NUMERIC, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_STRING, - "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_WIKITTY - }; - - /** solr server */ - protected SolrServer solrServer; - - /** Field modifier use to transform to solr format */ - protected TypeFieldModifier fieldModifier; - - /** JTA resource */ - protected SolrResource solrResource; - - /** - * Init wikitty search engine on solr embedded server. - * - * @param extensionStorage extension storage - * @param properties properties (can be null) - */ - public WikittySearchEngineSolrIndexInTreeNode( - ApplicationConfig config, WikittyExtensionStorage extensionStorage) { - - // init system env solr.data.dir - if (config != null) { - // choix du storage (file or Ram) - String solrDirFactoryKey = - WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_FACTORY.getKey(); - String solrDirFactory = config.getOption(solrDirFactoryKey); - if (solrDirFactory != null) { - System.setProperty(solrDirFactoryKey, solrDirFactory); - } - - // on utilise le directory que si on est pas en Ram - if (solrDirFactory != null && !solrDirFactory.contains("RAMDirectoryFactory")) { - String solrDataDirKey = - WikittyConfig.WikittyOption.WIKITTY_SEARCHENGINE_SOLR_DIRECTORY_DATA.getKey(); - String solrDataDir = config.getOption(solrDataDirKey); - // make sure that dir exists - if (solrDataDir != null) { - File file = new File(solrDataDir); - file.mkdirs(); - System.setProperty(solrDataDirKey, solrDataDir); - } - } - } - - try { - CoreContainer.Initializer initializer = new CoreContainer.Initializer(); - CoreContainer coreContainer = initializer.initialize(); - solrServer = new EmbeddedSolrServer(coreContainer, ""); - - fieldModifier = new TypeFieldModifier(extensionStorage); - solrResource = new SolrResource(solrServer); - - } catch (Exception eee) { - throw new WikittyException("SolR initialization error", eee); - } - } - - @Override - public void clear(WikittyTransaction transaction) { - try { - // FIXME poussin 20100618 pourquoi n'est pas fait dans la transaction ? - solrResource.init(); - solrServer.deleteByQuery("*:*"); - } catch (Exception eee) { - throw new WikittyException("Error during clearing SolR data", eee); - } - } - - @Override - public void store(WikittyTransaction transaction, Collection<Wikitty> wikitties) { - try { - solrResource.init(); - LinkedHashSet<String> treeNodeToIndex = new LinkedHashSet<String>(); - for (Wikitty w : wikitties) { - String id = w.getId(); - // Index - SolrInputDocument doc = createIndexDocument(w); - solrResource.addDoc(id, doc); - - if (WikittyTreeNodeHelper.hasExtension(w)) { - if (w.getDirty().contains(WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT) - || null == WikittyTreeNodeHelper.getParent(w) - || WikittyUtil.versionGreaterThan("1", w.getVersion())) { - // si le pere a change - // ou qu'il est null (creation d'un nouvel arbre) - // ou que l'objet n'a jamais ete sauve 1 > version - // il faut indexer le noeud - treeNodeToIndex.add(id); - } else { - // on recupere l'ancienne indexation - SolrDocument oldDoc = SolrUtil.findById(solrServer, id); - String root = (String)oldDoc.getFieldValue(TREENODE_ROOT); - Collection parents = oldDoc.getFieldValues(TREENODE_PARENTS); - - doc.addField(TREENODE_ROOT, root); - for (Object parent : parents) { - doc.addField(TREENODE_PARENTS, parent); - } - } - } - - } - - // Reindex child in tree node - reindexTreeNode(solrResource, treeNodeToIndex); - } catch (Exception eee) { - throw new WikittyException("Can't store wikitty", eee); - } - } - - protected String getTreeRoot(SolrResource solrResource, SolrServer solrServer, - String node) throws Exception { - String result = null; - String parent = node; - while (parent != null) { - SolrInputDocument doc = solrResource.getAddedDoc(parent); - if (doc != null) { - result = (String)doc.getFieldValue(TREENODE_ROOT); - parent = (String) doc.getFieldValue(WikittyTreeNode - .FQ_FIELD_WIKITTYTREENODE_PARENT + SUFFIX_WIKITTY); - } else { - SolrDocument parentDoc = SolrUtil.findById(solrServer, parent); - result = (String) parentDoc.getFirstValue(TREENODE_ROOT); - parent = (String) parentDoc.getFieldValue(WikittyTreeNode - .FQ_FIELD_WIKITTYTREENODE_PARENT + SUFFIX_WIKITTY); - } - } - // on a pas retrouve le root, cela veut dire que node et le root - if (result == null) { - result = node; - } - return result; - } - - /** - * Reindexe les noeuds passe en parametre et reindexe automatiquement leurs - * sous noeuds - * - * @param solrResource - * @param treeNodeToIndex - * @throws Exception - */ - protected void reindexTreeNode(SolrResource solrResource, - LinkedHashSet<String> treeNodeToIndex) throws Exception { - LinkedList<String> todo = new LinkedList<String>(treeNodeToIndex); - Set<String> done = new HashSet<String>(); - - Set<String> deleted = new HashSet<String>(solrResource.getDeletedDocs()); - - // key: child id, value: parent id - HashMap<String, String> tree = new HashMap<String, String>(); - while(todo.size() > 0) { - String id = todo.poll(); - if (done.contains(id)) { - continue; - } - done.add(id); - - // on recherche tout les fils du noeud que l'on reindex pour les reindexer aussi - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PARENTS + ":" + id); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - String childId = (String) solrDocument.getFieldValue(SOLR_ID); - todo.offer(childId); - } - - if (deleted.contains(id)) { - continue; - } - - // Get documents associated with this id - SolrInputDocument doc = solrResource.getAddedDoc(id); - if (doc == null) { - // Copy old field value - SolrDocument found = SolrUtil.findById(solrServer, id); - if (found != null) { - // copy only necessary field (other are regenerated) - doc = SolrUtil.copySolrDocument(found, true, fieldToCopyPattern); - solrResource.addDoc(id, doc); - } else { - if (log.isWarnEnabled()) { - log.warn("Can't find wikitty id '" + id + "' in index. Skip this wikitty."); - } - } - } - - // ajout du nouveau champs tree.root - String root = getTreeRoot(solrResource, solrServer, id); - doc.addField(TREENODE_ROOT, root); - - // ajout dans le nouveau champs tree.parents tous les parents du noeud - // et le noeud lui meme - String parent = id; - while (parent != null) { - String oldParent = parent; - doc.addField(TREENODE_PARENTS, parent); - if (tree.containsKey(parent)) { - parent = tree.get(parent); - } else if (null != solrResource.getAddedDoc(parent)) { - SolrInputDocument parentDoc = solrResource.getAddedDoc(id); - parent = (String) parentDoc.getFieldValue(WikittyTreeNode - .FQ_FIELD_WIKITTYTREENODE_PARENT + SUFFIX_WIKITTY); - } else { - SolrDocument parentDoc = SolrUtil.findById(solrServer, parent); - if (parentDoc != null) { - parent = (String) parentDoc.getFieldValue(WikittyTreeNode - .FQ_FIELD_WIKITTYTREENODE_PARENT + SUFFIX_WIKITTY); - } else { - parent = null; - } - } - tree.put(oldParent, parent); - } - } - } - - @Override - public void delete(WikittyTransaction transaction, Collection<String> ids) throws WikittyException { - try { - solrResource.init(); - LinkedHashSet<String> treeNodeToIndex = new LinkedHashSet<String>(); - for (String id : ids) { - SolrDocument doc = SolrUtil.findById(solrServer, id); - if (doc.containsKey(TREENODE_ROOT)) { - // child are automaticaly done in reindexTreeNode method - treeNodeToIndex.add(id); - } - solrResource.deleteDoc(id); - } - - // Reindex child in tree node - reindexTreeNode(solrResource, treeNodeToIndex); - } catch (Exception eee) { - throw new WikittyException("Can't delete wikitty in index", eee); - } - } - - @Override - public PagedResult<String> findAllByCriteria(WikittyTransaction transaction, Criteria criteria) { - try { - // Create query with restriction - Restriction2Solr restriction2Solr = new Restriction2Solr(transaction, fieldModifier); - String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); - - // Add paged - int firstIndex = criteria.getFirstIndex(); - int endIndex = criteria.getEndIndex(); - - query.setStart(firstIndex); - int nbRows; - if (endIndex == -1) { - // WARNING It is necessary to substract 'start' otherwise, - // there is a capacity overlow in solR - nbRows = Integer.MAX_VALUE - firstIndex; - } else { - nbRows = endIndex - firstIndex + 1; - } - query.setRows(nbRows); - - // Add sorting - List<String> sortAscending = criteria.getSortAscending(); - if(sortAscending != null) { - for (String sort : sortAscending) { - String tranform = fieldModifier.convertToSolr(transaction, sort); - query.addSortField(tranform, SolrQuery.ORDER.asc); - } - } - - List<String> sortDescending = criteria.getSortDescending(); - if(sortDescending != null) { - for (String sort : sortDescending) { - String tranform = fieldModifier.convertToSolr(transaction, sort); - query.addSortField(tranform, SolrQuery.ORDER.desc); - } - } - - // Add faceting - List<String> facetField = criteria.getFacetField(); - log.debug("facetField : " + facetField); - List<Criteria> facetCriteria = criteria.getFacetCriteria(); - - // use to map query string to criteria facet name - Map<String, String> facetQueryToName = new HashMap<String, String>(); - - if ((facetField != null && !facetField.isEmpty()) - || (facetCriteria != null && !facetCriteria.isEmpty())) { - query.setFacet(true); - query.setFacetMinCount(1); - // query.setFacetLimit(8); // no limit actualy - - // field facetisation - if (facetField != null) { - for (String fqfieldName : facetField) { - String tranform = fieldModifier.convertToSolr(transaction, fqfieldName); - query.addFacetField(tranform); - } - } - - // query facetisation - if (facetCriteria != null) { - for (Criteria facet : facetCriteria) { - String queryFacet = - restriction2Solr.toSolr(facet.getRestriction()); - facetQueryToName.put(queryFacet, facet.getName()); - query.addFacetQuery(queryFacet); - } - } - } - - if(log.isDebugEnabled()) { - log.debug(String.format("Try to execute query %s", query)); - } - QueryResponse resp = solrServer.query(query); - SolrDocumentList solrResults = resp.getResults(); - - Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); - if (facetField != null && !facetField.isEmpty()) { - for (FacetField facet : resp.getFacetFields()) { - String facetName = fieldModifier.convertToField(transaction, facet.getName()); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - if (facet.getValues() != null) { - for (FacetField.Count value : facet.getValues()) { - String topicName = value.getName(); -// if(!topicName.endsWith(TREENODE_EMPTY)) { - int topicCount = (int) value.getCount(); - FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); - topics.add(topic); -// } - } - } - facets.put(facetName, topics); - } - } - if (facetCriteria != null && !facetCriteria.isEmpty()) { - for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { - String facetName = facet.getKey(); - // don't use contains because, map can have key with null value - if (null != facetQueryToName.get(facetName)) { - facetName = facetQueryToName.get(facetName); - } - Integer count = facet.getValue(); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - FacetTopic topic = new FacetTopic(facetName, facetName, count); - topics.add(topic); - facets.put(facetName, topics); - } - } - - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue(SOLR_ID); - ids.add(id); - } - - int numFound = (int)resp.getResults().getNumFound(); - PagedResult<String> result = new PagedResult<String>( - firstIndex, numFound, queryString, facets, ids); - - return result; - } catch (SolrServerException eee) { - throw new WikittyException("Error during find", eee); - } - } - - /** - * Cette implantation n'est pas vraiment optimale en place memoire si - * le 'filter' retourne enormement de resultat ou qu'il y a beaucoup - * d'attachment dans l'arbre. - * - * On a en memoire l'ensemble des ids des attachments du sous arbre et - * l'ensemble des ids du resultat du filter. - * - * @param transaction - * @param w - * @param filter - * @return - */ - @Override - public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { - try { - int result = 0; - - String id = w.getId(); - // we used hashSet to count only once same attachments in many node - Collection allAttachments = new HashSet(); - - // on recherche tout les fils du noeud - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PARENTS + ":" + id); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - Collection attachments = solrDocument.getFieldValues( - WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT - + SUFFIX_WIKITTY); - if (attachments != null) { - allAttachments.addAll(attachments); - } - } - if (filter != null) { - Search search = Search.query(filter); - Search and = search.and().contains(SOLR_ID, allAttachments); - Criteria criteria = search.criteria(); - - PagedResult<String> criteriaResult = findAllByCriteria(transaction, filter); - allAttachments.retainAll(criteriaResult.getAll()); - } - result = allAttachments.size(); - return result; - } catch (SolrServerException eee) { - throw new WikittyException("Can't find node count", eee); - } - } - - /** - * Cette implantation n'est pas vraiment optimale en place memoire si - * le 'filter' retourne enormement de resultat ou qu'il y a beaucoup - * d'attachment dans l'arbre. - * - * On a en memoire l'ensemble des ids des attachments du sous arbre et - * l'ensemble des ids du resultat du filter. - * - * @param transaction - * @param w - * @param filter - * @return - */ - @Override - public Map<String, Integer> findAllChildrenCount( - WikittyTransaction transaction, Wikitty w, Criteria filter) { - try { - // key: id node; value: attachment count for this node (with sub*node) - Map<String, Integer> result = new HashMap<String, Integer>(); - - String id = w.getId(); - // key: id node; value: attachment count for this node (with sub*node) - Map<String, Collection> allAttachments = new HashMap<String, Collection>(); - - // les id des fils direct du wikitty passe en parametre - Set<String> child = new HashSet<String>(); - - // on recherche tout les fils et sous fils, lui compris - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PARENTS + ":" + id); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - String childId = (String) solrDocument.getFieldValue(SOLR_ID); - String parentId = (String) solrDocument.getFieldValue( - WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT + SUFFIX_WIKITTY); - Collection attachments = solrDocument.getFieldValues( - WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_ATTACHMENT + SUFFIX_WIKITTY); - Collection parents = solrDocument.getFieldValues( - TREENODE_PARENTS); - - if (id.equals(parentId)) { - // c'est un fils direct on l'ajoute a la liste des fils - child.add(childId); - } - - for (Object p : parents) { - String parent = (String) p; - Collection col = allAttachments.get(parent); - if (col == null) { - // we used hashSet to count only once same attachments in many node - col = new HashSet(); - allAttachments.put(parent, col); - } - if (attachments != null) { - col.addAll(attachments); - } - } - } - - // creation des id repondants au filtre si besoin - Collection<String> filteredId = Collections.EMPTY_LIST; - if (filter != null) { - - // stocke toutes les ids des attachments du sous arbres - Collection<String> attachmentIds = new LinkedList<String>(); - // 1ere passe, on ne garde que les fils directs. - // comme ca se sera deja fait pour la construction du result - // moins de chose a reparcourir - for (Iterator<Map.Entry<String, Collection>> i = allAttachments.entrySet().iterator(); i.hasNext();) { - Map.Entry<String, Collection> e = i.next(); - // on ne garde que les fils directs - if (child.contains(e.getKey())) { - attachmentIds.addAll(e.getValue()); - } else { - i.remove(); - } - } - - // ca ne sert a rien de faire la requete si au final, il n'y a - // aucun attachment - if (attachmentIds.size() > 0) { - Search search = Search.query(filter); - Search and = search.and().contains(SOLR_ID, attachmentIds); - Criteria criteria = search.criteria(); - - PagedResult<String> criteriaResult = findAllByCriteria(transaction, criteria); - filteredId = criteriaResult.getAll(); - } - } - - // Construction du resultat - for (Map.Entry<String, Collection> e : allAttachments.entrySet()) { - // on ne garde que les fils directs - // c'est peut-etre fait si filter est non null, mais pas sinon - if (child.contains(e.getKey())) { - Collection attachments = e.getValue(); - // on ne garde dans le comptage des attachments que les id - // qui reponde aussi au filter - attachments.retainAll(filteredId); - result.put(e.getKey(), attachments.size()); - } - } - - return result; - } catch (SolrServerException eee) { - throw new WikittyException("Can't find node count", eee); - } - - } - - /** - * Create all index document to used to modify indexation. - * this method don't modify index. - * - * The document looks like : - * SolrId : wikittyId - * extensions : extensionNames - * fieldName : fieldValue - * - * @param w all wikitties object to index - * @return solrInputDocument used to modify index - */ - protected SolrInputDocument createIndexDocument(Wikitty w) { - if (log.isDebugEnabled()) { - log.debug("index wikitty " + w.getId()); - } - - SolrInputDocument doc = new SolrInputDocument(); - String id = w.getId(); - doc.addField(SOLR_ID, id); - - for (String name : w.getExtensionNames()) { - doc.addField(SOLR_EXTENSIONS, name); - } - - for (String fqfieldName : w.fieldNames()) { - FieldType fieldType = w.getFieldType(fqfieldName); - TYPE type = fieldType.getType(); - String solrFqFieldName = SolrUtil.getSolrFieldName(fqfieldName, type); - - String solrAllFieldName = SOLR_ALL_EXTENSIONS - + WikittyUtil.FQ_FIELD_NAME_SEPARATOR - + WikittyUtil.getFieldNameFromFQFieldName(solrFqFieldName); - - Object objectValue = w.getFqField(fqfieldName); - if(objectValue != null) { - if (fieldType.isCollection()) { - Collection collectionValue = (Collection) objectValue; - for (Object itemValue : collectionValue) { - if (itemValue != null) { - doc.addField(solrFqFieldName, itemValue); - doc.addField(solrAllFieldName, itemValue); - doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - if (log.isDebugEnabled()) { - log.debug("index field " + solrFqFieldName + - " with value '" + itemValue + "'"); - } - } - } - } else { - doc.addField(solrFqFieldName, objectValue); - doc.addField(solrAllFieldName, objectValue); - doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - if (log.isDebugEnabled()) { - log.debug("index field " + solrFqFieldName + - " with value '" + objectValue + "'"); - } - } - } - } - return doc; - } - -} Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySolrConstant.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySolrConstant.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySolrConstant.java 2011-01-31 12:13:07 UTC (rev 699) @@ -48,13 +48,13 @@ /** id field in solr */ static final public String SOLR_ID = SOLR_WIKITTY_PREFIX + "id"; + /** if field is null, this extra field is set to true otherwize is set to false */ + static final public String SOLR_NULL_FIELD = + SOLR_WIKITTY_PREFIX + "null_field-"; + /** extensions field name in solr */ static final public String SOLR_EXTENSIONS = SOLR_WIKITTY_PREFIX + "extensions"; - /** group all fields is not null */ - static final public String SOLR_NOT_NULL_FIELDS = - SOLR_WIKITTY_PREFIX + "not_null_fields"; - /** extension use to store field without extension to search on all extesnion */ static final public String SOLR_ALL_EXTENSIONS = SOLR_WIKITTY_PREFIX + "all"; @@ -66,8 +66,16 @@ // Use for indexation tree node static final public String TREENODE_PREFIX = SOLR_WIKITTY_PREFIX + "tree."; + // Use as field on TreeNode static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; + // Use as field on TreeNode, contains parent node id and himself node id static final public String TREENODE_PARENTS = TREENODE_PREFIX + "parents"; + // Use as field on TreeNode, number of parents (root node depth=1) + static final public String TREENODE_DEPTH = TREENODE_PREFIX + "depth"; + // Use as field on Wikitty object attached on TreeNode, TreeNodeId is added at end + static final public String TREENODE_ATTACHED = TREENODE_PREFIX + "attached."; + // Use as field on Wikitty object attached on TreeNode, TreeNodeId is added at end + static final public String TREENODE_ATTACHED_ALL = TREENODE_PREFIX + "attached-all"; static final public String SUFFIX_BINARY = "_bi"; static final public String SUFFIX_BOOLEAN = "_b"; @@ -79,10 +87,4 @@ static final public String SUFFIX_STRING_LOWERCASE = "_c"; static final public String SUFFIX_STRING_FULLTEXT = "_t"; - - - // for old WikittySearchEngineSolr - static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; - static final public String TREENODE_PATH = TREENODE_PREFIX + "path"; - } Modified: trunk/wikitty-solr-impl/src/main/resources/schema.xml =================================================================== --- trunk/wikitty-solr-impl/src/main/resources/schema.xml 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/main/resources/schema.xml 2011-01-31 12:13:07 UTC (rev 699) @@ -44,6 +44,10 @@ <fieldType name="numeric" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/> + <!-- INT type --> + <fieldType name="int" class="solr.SortableIntField" + sortMissingLast="true" omitNorms="true"/> + <!-- DATE type --> <fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/> @@ -120,15 +124,15 @@ <!-- WARNING ALL DATA MUST BE STORED, WE MUST CAN REINDEX WIKITTY WITH INDEXED BECAUSE FOR HIEARCHICAL FACET, THERE ARE COPY OF DOCUMENT --> - <field name="#id" type="string" indexed="true" stored="true" required="true" /> + <field name="#id" type="string" required="true" indexed="true" stored="true" multiValued="false"/> <field name="#extensions" type="string" indexed="true" stored="true" multiValued="true"/> - <field name="#not_null_fields" type="string" indexed="true" stored="true" multiValued="true"/> - <field name="#tree.root" type="string" indexed="true" stored="false" multiValued="true"/> - <field name="#tree.parents" type="string" indexed="true" stored="false" multiValued="true"/> + <field name="#tree.root" type="string" indexed="true" stored="true" multiValued="false"/> + <field name="#tree.parents" type="string" indexed="true" stored="true" multiValued="true"/> + <field name="#tree.depth" type="int" indexed="true" stored="true" multiValued="false"/> + <field name="#tree.attached-all" type="string" indexed="true" stored="false" multiValued="true"/> + <dynamicfield name="#tree.attached.*" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicfield name="#null_field-*" type="boolean" indexed="true" stored="true" multiValued="false"/> - <!-- need for WikittySearchEngineSolr implementation --> - <dynamicfield name="#tree*" type="string" indexed="true" stored="false" multiValued="true"/> - <!-- copy all field (except binary) in '#fulltext' field for fulltext search --> <field name="#fulltext" type="text" indexed="true" stored="false" multiValued="true"/> <!-- copied field not stored --> @@ -160,6 +164,7 @@ <!-- SolrQueryParser configuration: defaultOperator="AND|OR" --> <solrQueryParser defaultOperator="AND"/> + <copyField source="#tree.attached.*" dest="#tree.attached-all"/> <copyField source="*_b" dest="#fulltext"/> <copyField source="*_d" dest="#fulltext"/> Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java 2011-01-31 12:13:07 UTC (rev 699) @@ -30,10 +30,11 @@ import static junit.framework.Assert.assertNotNull; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import org.apache.commons.logging.Log; @@ -43,7 +44,6 @@ import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.entities.ExtensionFactory; import org.nuiton.wikitty.entities.FieldType.TYPE; -import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.entities.WikittyTreeNode; import org.nuiton.wikitty.entities.WikittyTreeNodeImpl; import org.nuiton.wikitty.entities.Wikitty; @@ -53,6 +53,7 @@ import org.nuiton.wikitty.conform.StorageTest; import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.search.TreeNodeResult; /** * @@ -169,48 +170,83 @@ return wikitty; } - /** - * Count all element in sub tree, with element in node - */ - protected int sum(Wikitty node) { - int sum = 0; + static private class CollectAttachmentVisitor implements TreeNodeResult.Visitor<WikittyTreeNode> { - // Sum value in node - Set<String> values = WikittyTreeNodeHelper.getAttachment(node); - if(values != null) { - sum = values.size(); + protected Set attachment = new HashSet(); + + public Set getAttachment() { + return attachment; } - // Sum children node in node - String nodeId = node.getId(); - Map<WikittyTreeNode, Integer> children = getProxy().restoreChildren( - WikittyTreeNode.class, nodeId, null); - - /* - for (Integer count : children.values()) { - sum += count; + @Override + public boolean visitEnter(TreeNodeResult<WikittyTreeNode> node) { + Collection att = node.getObject().getAttachment(); + if (att != null) { + attachment.addAll(att); + } + return true; } - */ - - for (Map.Entry<WikittyTreeNode, Integer> e : children.entrySet()) { - log.debug("*treeNode = " + e.getKey().getName() + " " + e.getValue() + - " -> " + e.getKey().getAttachment()); + + @Override + public boolean visitLeave(TreeNodeResult<WikittyTreeNode> node) { + return true; } - - for (WikittyTreeNode treeNode : children.keySet()) { - Set<String> treeNodeChildren = treeNode.getAttachment(); - log.debug("+treeNode = " + treeNode.getName() + " " + (treeNodeChildren==null?0:treeNodeChildren.size()) + - " -> " + treeNodeChildren); -// if (treeNodeChildren == null) { -// sum += 0; -// } else { -// sum += treeNodeChildren.size(); -// } - sum += sum(((WikittyTreeNodeImpl)treeNode).getWikitty()); - } + } - return sum; + /** + * Count all element in sub tree, with element in node + */ + protected int sum(Wikitty node) { + // Sum attachment object in node + String nodeId = node.getId(); + TreeNodeResult<WikittyTreeNode> tree = getProxy().findTreeNode( + nodeId, -1, true, null); + CollectAttachmentVisitor visitor = new CollectAttachmentVisitor(); + tree.acceptVisitor(visitor); + + int result=visitor.getAttachment().size(); + return result; } +// int sum = 0; +// +// // Sum value in node +// Set<String> values = WikittyTreeNodeHelper.getAttachment(node); +// if(values != null) { +// sum = values.size(); +// } +// +// // Sum children node in node +// String nodeId = node.getId(); +// TreeNodeResult<WikittyTreeNode> children = getProxy().findTreeNode( +// nodeId, 1, true, null); +// +// /* +// for (Integer count : children.values()) { +// sum += count; +// } +// */ +// +// for (TreeNodeResult<WikittyTreeNode> e : children.getChildren()) { +// log.debug("*treeNode = " + e.getObject().getName() + " " +// + e.getAttCount() + " -> " + e.getObject().getAttachment()); +// } +// +// for (TreeNodeResult<WikittyTreeNode> treeNodeResult : children) { +// WikittyTreeNode treeNode = treeNodeResult.getObject(); +// Set<String> treeNodeChildren = treeNode.getAttachment(); +// log.debug("+treeNode = " + treeNode.getName() + " " + +// (treeNodeChildren==null?0:treeNodeChildren.size()) + +// " -> " + treeNodeChildren); +//// if (treeNodeChildren == null) { +//// sum += 0; +//// } else { +//// sum += treeNodeChildren.size(); +//// } +// sum += sum(((WikittyTreeNodeImpl)treeNode).getWikitty()); +// } +// +// return sum; +// } /** * Create a Wikitty WikittyTreeNode @@ -237,7 +273,8 @@ Wikitty root = findNode("root"); String rootId = root.getId(); - WikittyTree tree = getProxy().restoreTree(rootId); + TreeNodeResult<WikittyTreeNode> tree = + getProxy().findTreeNode(rootId, -1, false, null); assertNotNull(tree); } @@ -261,9 +298,9 @@ Wikitty node1 = findNode("node1"); String node1Id = node1.getId(); - Map<WikittyTreeNode, Integer> children = getProxy().restoreChildren( - WikittyTreeNode.class, node1Id, null); - assertEquals(3, children.size()); + TreeNodeResult<WikittyTreeNode> children = getProxy().findTreeNode( + node1Id, 1, false, null); + assertEquals(3, children.getChildCount()); } @Test @@ -277,12 +314,12 @@ String node1Id = node1.getId(); Criteria filter = Search.query().eq("test.name", "value 3").criteria(); - Map<WikittyTreeNode, Integer> children = getProxy().restoreChildren( - WikittyTreeNode.class, node1Id, filter); + TreeNodeResult<WikittyTreeNode> children = getProxy().findTreeNode( + node1Id, 1, true, filter); System.out.println(children); - assertEquals(3, children.size()); - for (Map.Entry<WikittyTreeNode, Integer> e : children.entrySet()) { - assertEquals(result.get(e.getKey().getName()), e.getValue()); + assertEquals(3, children.getChildCount()); + for (TreeNodeResult<WikittyTreeNode> e : children.getChildren()) { + assertEquals(result.get(e.getObject().getName()), Integer.valueOf(e.getAttCount())); } } @@ -291,9 +328,9 @@ Wikitty node11 = findNode("node11"); String node11Id = node11.getId(); - Entry<WikittyTreeNode, Integer> count = getProxy().restoreNode( - WikittyTreeNode.class, node11Id, null); - assertEquals(3, count.getValue().intValue()); + TreeNodeResult<WikittyTreeNode> count = getProxy().findTreeNode( + node11Id, 0, true, null); + assertEquals(3, count.getAttCount()); } @Test @@ -302,10 +339,10 @@ String node11Id = node11.getId(); Criteria filter = Search.query().eq("test.name", "value 3").criteria(); - Entry<WikittyTreeNode, Integer> count = getProxy().restoreNode( - WikittyTreeNode.class, node11Id, filter); + TreeNodeResult<WikittyTreeNode> count = getProxy().findTreeNode( + node11Id, 0, true, filter); System.out.println(count); - assertEquals(1, count.getValue().intValue()); + assertEquals(1, count.getAttCount()); } @Test @@ -313,9 +350,9 @@ // Check that node 2 it has any child Wikitty node2 = findNode("node2"); String node2Id = node2.getId(); - Map<WikittyTreeNode, Integer> children = getProxy().restoreChildren( - WikittyTreeNode.class, node2Id, null); - assertEquals(0, children.size()); + TreeNodeResult<WikittyTreeNode> children = getProxy().findTreeNode( + node2Id, 1, true, null); + assertEquals(0, children.getChildCount()); // Create a new node, child of node 2 Wikitty nodeWikitty = createNode("node21", node2Id); @@ -326,9 +363,9 @@ assertNotNull(found); // Check that it was great added as node2 child - children = getProxy().restoreChildren( - WikittyTreeNode.class, node2Id, null); - assertEquals(1, children.size()); + children = getProxy().findTreeNode( + node2Id, 1, false, null); + assertEquals(1, children.getChildCount()); } @Test @@ -425,20 +462,20 @@ WikittyEvent event = ws.store(null, child.getWikitty()); event.update(child.getWikitty()); - Map<WikittyTreeNode, Integer> children = getProxy().restoreChildren( - WikittyTreeNode.class, parent.getWikittyId(), null); + TreeNodeResult<WikittyTreeNode> children = getProxy().findTreeNode( + parent.getWikittyId(), 1, true, null); - assertEquals(1, children.size()); - assertEquals(0, children.get(child).intValue()); + assertEquals(1, children.getChildCount()); + assertEquals(0, children.getChild(child).getAttCount()); child.setParent(null); ws.store(null, child.getWikitty()); - children = getProxy().restoreChildren( - WikittyTreeNode.class, parent.getWikittyId(), null); + children = getProxy().findTreeNode( + parent.getWikittyId(), 1, true, null); - assertEquals(0, children.size()); + assertEquals(0, children.getChildCount()); } @Test @@ -503,11 +540,11 @@ Wikitty node3 = findNode("node3"); int sum = sum(node3); - assertEquals(2, sum); + assertEquals(1, sum); - Entry<WikittyTreeNode, Integer> count = getProxy().restoreNode( - WikittyTreeNode.class, node3.getId(), null); - assertEquals(1, count.getValue().intValue()); + TreeNodeResult<WikittyTreeNode> count = getProxy().findTreeNode( + node3.getId(), 0, true, null); + assertEquals(1, count.getAttCount()); sum = sum(node31); assertEquals(1, sum); Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrTest.java 2011-01-31 11:56:44 UTC (rev 698) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrTest.java 2011-01-31 12:13:07 UTC (rev 699) @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.Assert; @@ -42,6 +41,7 @@ import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.search.PagedResult; import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.search.TreeNodeResult; /** * @@ -54,7 +54,7 @@ public class WikittySearchEngineSolrTest { /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(WikittySearchEngineSolrTest.class); + private static final Log log = LogFactory.getLog(WikittySearchEngineSolrTest.class); protected WikittyServiceSolr ws =new WikittyServiceSolr(new WikittyConfig()); @@ -177,15 +177,22 @@ ws.store(null, toStore, false); toStore.clear(); - Integer val = ws.getSearchEngine().findNodeCount(null, root, null); - Assert.assertEquals(Integer.valueOf(12), val); + TreeNodeResult<String> treeNodeResult = + ws.getSearchEngine().findAllChildrenCount( + null, root.getId(), 0, true, null); + int val = treeNodeResult.getAttCount(); +// Integer val = ws.getSearchEngine().findNodeCount(null, root, null); + Assert.assertEquals(12, val); -// Map<WikittyTreeNode, Integer> children = ws.restoreChildren(null, root.getId(), null); - Map<String, Integer> children = ws.getSearchEngine().findAllChildrenCount(null, root, null); +// Map<WikittyTreeNode, Integer> children = ws.findTreeNode(null, root.getId(), null); +// Map<String, Integer> children = ws.getSearchEngine().findAllChildrenCount(null, root, null); + TreeNodeResult<String> children = + ws.getSearchEngine().findAllChildrenCount( + null, root.getId(), 1, true, null); System.out.println(children); - Assert.assertEquals(2, children.size()); - Assert.assertEquals(Integer.valueOf(5), children.get(node1.getId())); - Assert.assertEquals(Integer.valueOf(4), children.get(node2.getId())); + Assert.assertEquals(2, children.getChildCount()); + Assert.assertEquals(5, children.getChild(node1.getId()).getAttCount()); + Assert.assertEquals(4, children.getChild(node2.getId()).getAttCount()); } }