- Wikitty-commits - mailman.cloud.codelutin.com

r736 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 18:20:17 +0100 (Wed, 23 Feb 2011) New Revision: 736 Url: http://nuiton.org/repositories/revision/wikitty/736 Log: prevent loop in TreeNode parent search algo Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 16:59:49 UTC (rev 735) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 17:20:17 UTC (rev 736) @@ -491,7 +491,8 @@ SolrInputDocument doc, Map<String, SolrDocument> tree) { Set<String> parents = new HashSet<String>(); String root = null; - String parentId = (String)doc.getFieldValue(SOLR_ID); + String treeNodeId = (String)doc.getFieldValue(SOLR_ID); + String parentId = treeNodeId; if (parentId == null) { throw new WikittyException("parentId is null, but this must be impossible"); } @@ -528,8 +529,16 @@ } } if (nextParentId != null) { - parents.add(nextParentId); - parentId = nextParentId; + if (parents.contains(nextParentId)) { + log.error(String.format("Tree with TreeNode '%s' have loop" + + " at node %s->%s all parents are %s. Set root with" + + " last valide parent '%s'", + treeNodeId, parentId, nextParentId, parents, parentId)); + root = parentId; + } else { + parents.add(nextParentId); + parentId = nextParentId; + } } else { root = parentId; }

1 0

r735 - trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 17:59:49 +0100 (Wed, 23 Feb 2011) New Revision: 735 Url: http://nuiton.org/repositories/revision/wikitty/735 Log: error in log message (nb) Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2011-02-23 16:42:49 UTC (rev 734) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2011-02-23 16:59:49 UTC (rev 735) @@ -985,9 +985,9 @@ wikitties.clear(); tx.begin(); timeLog.log(startStore, "StoreAndCommit", - String.format("nb %s", count)); + String.format("nb %s", numberForCommit)); startLoadStoreCommit = timeLog.log(startLoadStoreCommit, - "LoadAndStoreAndCommit", String.format("nb %s", count)); + "LoadAndStoreAndCommit", String.format("nb %s", numberForCommit)); // si on a les statistiques a disposition on affiche // l'etat d'avancement dans les logs if (max > 0) {

1 0

r734 - trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 17:42:49 +0100 (Wed, 23 Feb 2011) New Revision: 734 Url: http://nuiton.org/repositories/revision/wikitty/734 Log: add TimeLog info in syncSearchEngine Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2011-02-23 16:42:29 UTC (rev 733) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2011-02-23 16:42:49 UTC (rev 734) @@ -40,6 +40,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.util.TimeLog; import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.entities.FieldType; import org.nuiton.wikitty.search.PagedResult; @@ -66,7 +67,8 @@ public class WikittyServiceStorage implements WikittyService { /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(WikittyServiceStorage.class); + final static private Log log = LogFactory.getLog(WikittyServiceStorage.class); + final static private TimeLog timeLog = new TimeLog(WikittyServiceStorage.class); /** Default migration use to migrate a wikitty in last extension version */ protected WikittyExtensionMigration defaultExtensionMigration = @@ -962,6 +964,7 @@ long lastPurcent = -1; long current = 0; int count = 0; + long startLoadStoreCommit = TimeLog.getTime(); @Override public void scan(String wikittyId) { @@ -974,12 +977,17 @@ if(count == numberForCommit) { // Reindex + long startStore = TimeLog.getTime(); searchEngine.store(tx, wikitties, true); tx.commit(); // Reinit count = 0; wikitties.clear(); tx.begin(); + timeLog.log(startStore, "StoreAndCommit", + String.format("nb %s", count)); + startLoadStoreCommit = timeLog.log(startLoadStoreCommit, + "LoadAndStoreAndCommit", String.format("nb %s", count)); // si on a les statistiques a disposition on affiche // l'etat d'avancement dans les logs if (max > 0) {

1 0

r733 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 17:42:29 +0100 (Wed, 23 Feb 2011) New Revision: 733 Url: http://nuiton.org/repositories/revision/wikitty/733 Log: remove unused import Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 16:29:00 UTC (rev 732) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 16:42:29 UTC (rev 733) @@ -63,7 +63,6 @@ import java.util.Collections; import org.apache.commons.lang.StringUtils; import org.nuiton.util.ApplicationConfig; -import org.nuiton.util.StringUtil; import org.nuiton.util.TimeLog; import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.WikittyUtil;

1 0

r732 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 17:29:00 +0100 (Wed, 23 Feb 2011) New Revision: 732 Url: http://nuiton.org/repositories/revision/wikitty/732 Log: remove double same condition Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 16:28:11 UTC (rev 731) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 16:29:00 UTC (rev 732) @@ -501,18 +501,16 @@ String nextParentId = null; SolrInputDocument parentDoc = solrResource.getAddedDoc(parentId); if (parentDoc != null) { - if (parentDoc != null) { - // si parentDoc a deja ete indexe pour l'arbre, on peut reutiliser - // directement les valeurs et sortir de la boucle - if (parentDoc.containsKey(TREENODE_ROOT)) { - root = (String) parentDoc.getFieldValue(TREENODE_ROOT); - Collection p = parentDoc.getFieldValues(TREENODE_PARENTS); - parents.addAll(p); - break; - } else { - nextParentId = (String) parentDoc.getFieldValue(SolrUtil.getSolrFieldName( - WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, TYPE.WIKITTY)); - } + // si parentDoc a deja ete indexe pour l'arbre, on peut reutiliser + // directement les valeurs et sortir de la boucle + if (parentDoc.containsKey(TREENODE_ROOT)) { + root = (String) parentDoc.getFieldValue(TREENODE_ROOT); + Collection p = parentDoc.getFieldValues(TREENODE_PARENTS); + parents.addAll(p); + break; + } else { + nextParentId = (String) parentDoc.getFieldValue(SolrUtil.getSolrFieldName( + WikittyTreeNode.FQ_FIELD_WIKITTYTREENODE_PARENT, TYPE.WIKITTY)); } } else { SolrDocument oldParentDoc = tree.get(parentId);

1 0

r731 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 17:28:11 +0100 (Wed, 23 Feb 2011) New Revision: 731 Url: http://nuiton.org/repositories/revision/wikitty/731 Log: add assert in algo Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:53:47 UTC (rev 730) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 16:28:11 UTC (rev 731) @@ -493,6 +493,9 @@ Set<String> parents = new HashSet<String>(); String root = null; String parentId = (String)doc.getFieldValue(SOLR_ID); + if (parentId == null) { + throw new WikittyException("parentId is null, but this must be impossible"); + } parents.add(parentId); while (root == null) { String nextParentId = null;

1 0

r730 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 16:53:47 +0100 (Wed, 23 Feb 2011) New Revision: 730 Url: http://nuiton.org/repositories/revision/wikitty/730 Log: remove some log debug, and some context value in exception message Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:51:35 UTC (rev 729) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:53:47 UTC (rev 730) @@ -661,7 +661,6 @@ // Add faceting List<String> facetField = criteria.getFacetField(); - log.debug("facetField : " + facetField); List<Criteria> facetCriteria = criteria.getFacetCriteria(); // use to map query string to criteria facet name @@ -692,9 +691,6 @@ } } - if(log.isDebugEnabled()) { - log.debug(String.format("Try to execute query %s", query)); - } QueryResponse resp = SolrUtil.executeQuery(solrServer, query); SolrDocumentList solrResults = resp.getResults(); @@ -741,7 +737,8 @@ return result; } catch (SolrServerException eee) { - throw new WikittyException("Error during find", eee); + throw new WikittyException(String.format( + "Error during find of criteria '%s'", criteria), eee); } }

1 0

r729 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 16:51:35 +0100 (Wed, 23 Feb 2011) New Revision: 729 Url: http://nuiton.org/repositories/revision/wikitty/729 Log: change some log level from debug to trace, and abbreviate fieldvalue Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:40:54 UTC (rev 728) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:51:35 UTC (rev 729) @@ -61,7 +61,9 @@ import java.io.File; import java.util.Collections; +import org.apache.commons.lang.StringUtils; import org.nuiton.util.ApplicationConfig; +import org.nuiton.util.StringUtil; import org.nuiton.util.TimeLog; import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.WikittyUtil; @@ -877,9 +879,10 @@ doc.addField(solrFqFieldName, fieldValue); doc.addField(solrAllFieldName, fieldValue); solrNullFieldFqFieldNameValue = "false"; - if (log.isDebugEnabled()) { - log.debug("index field " + solrFqFieldName - + " with value '" + fieldValue + "'"); + if (log.isTraceEnabled()) { + log.trace(String.format("index field '%s' with value '%s'", + solrFqFieldName, + StringUtils.abbreviate(String.valueOf(fieldValue), 50))); } } doc.addField(solrNullFieldFqFieldName, solrNullFieldFqFieldNameValue); @@ -927,8 +930,8 @@ * @return solrInputDocument used to modify index */ protected SolrInputDocument createIndexDocument(Wikitty w) { - if (log.isDebugEnabled()) { - log.debug("index wikitty " + w.getId()); + if (log.isTraceEnabled()) { + log.trace(String.format("create index wikitty for '%s'", w.getId())); } SolrInputDocument doc = new SolrInputDocument();

1 0

r728 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 16:40:54 +0100 (Wed, 23 Feb 2011) New Revision: 728 Url: http://nuiton.org/repositories/revision/wikitty/728 Log: change store indexation for attachment if not present in Storage or in Transaction. Create partial document for futur storage of this document Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:40:00 UTC (rev 727) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolr.java 2011-02-23 15:40:54 UTC (rev 728) @@ -164,9 +164,6 @@ Map<String, Wikitty> allWikitties = new HashMap<String, Wikitty>(); // les ids des wikitties en parametre reellement modifier (a reindexer) Set<String> dirtyObject = new HashSet<String>(); - // les ids des wikitties qui n'appartiennent pas a un TreeNode passé en - // parametre, mais qui pourrait appartenir a un treenode deja indexe - Set<String> possibleAttachment = new HashSet<String>(); // les ids des TreeNodes dont le champs parent a change (est aussi // contenu dans dirtyObject Set<String> dirtyParent = new HashSet<String>(); @@ -220,13 +217,14 @@ Wikitty w = allWikitties.get(id); SolrDocument oldDoc = dirtyObjectDoc.get(id); SolrInputDocument doc = createIndexDocument(w); - if (oldDoc == null) { - // Le document n'existait pas, mais peut-etre qu'il etait - // dans la liste des attachments d'un TreeNode. Il faudra - // donc rechercher si c'est le cas et l'indexer convenablement - possibleAttachment.add(id); - } else { - // copy des champs #tree.attached + if (oldDoc != null) { + // On a un ancien document partiel ou complet + // s'il etait partiel (seulement l'indexation arbre + // cela veut dire que l'objet TreeNode a ete store + // sans que l'attachment le soit et qu'il l'est que + // maintenant + + // copy des champs #tree.attached des documents partiels ou non SolrUtil.copySolrDocument(oldDoc, doc, TREENODE_ATTACHED + ".*"); if (WikittyTreeNodeHelper.hasExtension(w) && !dirtyParentDoc.containsKey(id)) { @@ -326,15 +324,7 @@ // Phase 3: on reindexe les attachments qui en ont besoin // - // on ajoute les attachment au TreeNode qui ont ete store, sans - // leurs attachments (ni en base, ni dans le meme store) - Map<String, SolrDocument> tree = - SolrUtil.findAllByAttachment(solrServer, possibleAttachment); - for (SolrDocument doc : tree.values()) { - attachmentInTree.add(doc, possibleAttachment); - } - - addTreeIndexField(solrResource, tree, attachmentInTree); + addTreeIndexField(solrResource, null, attachmentInTree); } catch (Exception eee) { throw new WikittyException("Can't store wikitty", eee); } @@ -605,20 +595,23 @@ // reindexation totale). Ceci n'est pas un probleme car // les attachments seront convenablement indexes lorsqu'ils // seront ajoutes - if (oldDoc != null || doc != null) { - if (doc == null) { - doc = new SolrInputDocument(); - SolrUtil.copySolrDocument(oldDoc, doc); - solrResource.addDoc(attId, doc); - } - doc.removeField(TREENODE_ATTACHED + treeNodeId); - for (Object id : treeNodeParents) { - doc.addField(TREENODE_ATTACHED + treeNodeId, id); - } - } else { - // FIXME, il faut tout de meme creer le document - // avec seulement l'id et l'indexation arbre + if (oldDoc == null && doc == null) { + // L'objet en attachment du TreeNode, n'est pas encore + // store et n'est pas dans le meme appel de la methode + // store. On cree donc un document Partiel (seulement + // constitue de l'id et de l'indexation d'arbre qui sera + // reutilise lors du store de reel objet + doc = new SolrInputDocument(); + addToIndexDocument(doc, null, SOLR_ID, attId); + } else if (doc == null) { + doc = new SolrInputDocument(); + SolrUtil.copySolrDocument(oldDoc, doc); + solrResource.addDoc(attId, doc); } + doc.removeField(TREENODE_ATTACHED + treeNodeId); + for (Object id : treeNodeParents) { + doc.addField(TREENODE_ATTACHED + treeNodeId, id); + } } } }

1 0

r727 - trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc
by bpoussin＠users.nuiton.org Feb. 23, 2011

Feb. 23, 2011

Author: bpoussin Date: 2011-02-23 16:40:00 +0100 (Wed, 23 Feb 2011) New Revision: 727 Url: http://nuiton.org/repositories/revision/wikitty/727 Log: remove double ;; Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java =================================================================== --- trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java 2011-02-23 15:18:43 UTC (rev 726) +++ trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java 2011-02-23 15:40:00 UTC (rev 727) @@ -626,7 +626,7 @@ @Override public DataStatistic getDataStatistic(WikittyTransaction transaction) { - DataStatistic result;; + DataStatistic result; Connection connection = WikittyJDBCUtil.getConnection(config); try { Statement statement = connection.createStatement();

1 0