November 2009 - Sandbox-commits - mailman.cloud.codelutin.com

r317 - in wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src: main/java/org/sharengo/wikitty/jdbc test/java/org/sharengo/wikitty/jdbc test/java/org/sharengo/wikitty/test/util test/resources
by kmorin＠users.nuiton.org 10 Nov '09

10 Nov '09

Author: kmorin Date: 2009-11-10 17:30:58 +0100 (Tue, 10 Nov 2009) New Revision: 317 Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/test/util/GenTestData.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/log4j.properties Log: config solr Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java 2009-11-10 16:30:50 UTC (rev 316) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java 2009-11-10 16:30:58 UTC (rev 317) @@ -56,19 +56,19 @@ */ static private Log log = LogFactory.getLog(WikittyServiceJDBC.class); - protected WikittySearchEngin searchEngin; + protected WikittySearchEngine searchEngine; protected WikittyExtensionStorage extensionStorage; protected WikittyStorage wikittyStorage; public WikittyServiceJDBC() { extensionStorage = new WikittyExtensionStorageJDBC(); wikittyStorage = new WikittyStorageJDBC(extensionStorage); - searchEngin = new WikittySearchEngineSolr(extensionStorage); + searchEngine = new WikittySearchEngineSolr(extensionStorage); } @Override - protected WikittySearchEngin getSearchEngin() { - return searchEngin; + protected WikittySearchEngine getSearchEngine() { + return searchEngine; } @Override Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java 2009-11-10 16:30:50 UTC (rev 316) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java 2009-11-10 16:30:58 UTC (rev 317) @@ -21,6 +21,7 @@ import org.sharengo.wikitty.Criteria; import org.sharengo.wikitty.Wikitty; import org.sharengo.wikitty.WikittyService; +import org.sharengo.wikitty.search.Element; import org.sharengo.wikitty.search.Search; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; @@ -36,7 +37,7 @@ public void testExport() throws Exception { final List<Wikitty> wikitties = createSampleWikitty(ws); - Criteria criteria = Search.query().eqExt(EXTNAME).criteria(); + Criteria criteria = Search.query().eq(Element.ELT_EXTENSION, EXTNAME).criteria(); String result = ws.syncExportAllByCriteria(criteria); log.debug("[XML] " + result); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java 2009-11-10 16:30:50 UTC (rev 316) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java 2009-11-10 16:30:58 UTC (rev 317) @@ -26,16 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.sharengo.internal.TreeNodeImpl; -import org.sharengo.wikitty.Criteria; -import org.sharengo.wikitty.ExtensionFactory; -import org.sharengo.wikitty.FacetTopic; -import org.sharengo.wikitty.PagedResult; -import org.sharengo.wikitty.Tree; -import org.sharengo.wikitty.TreeNode; -import org.sharengo.wikitty.Wikitty; -import org.sharengo.wikitty.WikittyExtension; -import org.sharengo.wikitty.WikittyService; +import org.sharengo.wikitty.*; import org.sharengo.wikitty.FieldType.TYPE; import org.sharengo.wikitty.search.Search; import org.sharengo.wikitty.solr.WikittySearchEngineSolr; @@ -96,8 +87,8 @@ Wikitty found = findNode(name); if(found == null) { TreeNodeImpl node = new TreeNodeImpl(); - node.setTreeNodeName(name); - node.setTreeNodeParent(parent); + node.setName(name); + node.setParent(parent); found = node.getWikitty(); log.debug("[Storing] " + name + " with id " + found.getId()); Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java 2009-11-10 16:30:50 UTC (rev 316) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java 2009-11-10 16:30:58 UTC (rev 317) @@ -110,7 +110,7 @@ public void setUp() throws Exception { // import data and prepare expected result ws = new WikittyServiceJDBC(); - ((WikittyServiceJDBC)ws).getSearchEngin().clear(); + ((WikittyServiceJDBC)ws).getSearchEngine().clear(); assertReference = new AssertReference(); GenTestData.importGeneratedFile( GenTestData.getDataFile("selected-data.txt"), ws, assertReference); Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/test/util/GenTestData.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/test/util/GenTestData.java 2009-11-10 16:30:50 UTC (rev 316) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/test/util/GenTestData.java 2009-11-10 16:30:58 UTC (rev 317) @@ -17,6 +17,9 @@ import org.sharengo.wikitty.WikittyExtension; import org.sharengo.wikitty.WikittyService; import org.sharengo.wikitty.WikittyUtil; +import org.sharengo.wikitty.jdbc.WikittyServiceJDBC; +import org.sharengo.wikitty.search.Element; +import org.sharengo.wikitty.search.Search; public class GenTestData { @@ -152,7 +155,11 @@ } public static void main(String[] args) throws Exception { - File file = getDataFile( "random-data.txt" ); - new GenTestData( file ); +// File file = getDataFile( "random-data.txt" ); +// new GenTestData( file ); + + WikittyService ws = new WikittyServiceJDBC(); + ws.findAllByCriteria(Search.query().eq("name","*test*").criteria()); + } } Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/log4j.properties =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/log4j.properties 2009-11-10 16:30:50 UTC (rev 316) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/log4j.properties 2009-11-10 16:30:58 UTC (rev 317) @@ -6,4 +6,4 @@ # Configuration by components log4j.rootLogger=ERROR, logConsole log4j.category.org.sharengo.wikitty=DEBUG -#log4j.category.org.apache.solr=DEBUG +log4j.category.org.apache.solr=DEBUG

1 0

r316 - in wikengo_core-wikitty/wikengo_core-wikitty-api/src: main/java/org/sharengo/wikitty test/resources
by kmorin＠users.nuiton.org 10 Nov '09

10 Nov '09

Author: kmorin Date: 2009-11-10 17:30:50 +0100 (Tue, 10 Nov 2009) New Revision: 316 Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/AbstractWikittyService.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyProxy.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyService.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyServiceInMemory.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/test/resources/log4j.properties Log: config solr Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/AbstractWikittyService.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/AbstractWikittyService.java 2009-11-06 18:01:21 UTC (rev 315) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/AbstractWikittyService.java 2009-11-10 16:30:50 UTC (rev 316) @@ -54,7 +54,7 @@ /** * Abstract class that new implementation must extends. * New implementation only have three method to implement: - * <li>getSearchEngin + * <li>getSearchEngine * <li>getExtensionStorage * <li>getWikittyStorage * @@ -85,7 +85,7 @@ protected Map<String, Future<String>> importExportTask = new HashMap<String, Future<String>>(); - abstract protected WikittySearchEngin getSearchEngin(); + abstract protected WikittySearchEngine getSearchEngine(); abstract protected WikittyExtensionStorage getExtensionStorage(); abstract protected WikittyStorage getWikittyStorage(); @@ -136,8 +136,8 @@ getWikittyStorage().prepare(transaction, wikitties, disableAutoVersionIncrement); // create solr wikitty document for solr - List<WikittySearchEngin.Command> wikittyIndexationCommandList = - getSearchEngin().prepare(transaction, wikitties); + List<WikittySearchEngine.Command> wikittyIndexationCommandList = + getSearchEngine().prepare(transaction, wikitties); // All prepare (store and index) are good // try to commit command @@ -145,7 +145,7 @@ extensionStorageCommandList); UpdateResponse wikUpdate = getWikittyStorage().commit(transaction, wikittyStorageCommandList); - UpdateResponse indexUpdate = getSearchEngin().commit(transaction, + UpdateResponse indexUpdate = getSearchEngine().commit(transaction, wikittyIndexationCommandList); UpdateResponse result = new UpdateResponse(); @@ -245,16 +245,16 @@ WikittyTransaction transaction = new WikittyTransaction(); List<WikittyStorage.Command> storeCommands = getWikittyStorage().delete(idList); - List<WikittySearchEngin.Command> indexCommands = - getSearchEngin().delete(transaction, idList); + List<WikittySearchEngine.Command> indexCommands = + getSearchEngine().delete(transaction, idList); getWikittyStorage().commit(transaction, storeCommands); - getSearchEngin().commit(transaction, indexCommands); + getSearchEngine().commit(transaction, indexCommands); } public PagedResult<Wikitty> findAllByCriteria(Criteria criteria) { - PagedResult<String> resultId = getSearchEngin().findAllByCriteria(criteria); + PagedResult<String> resultId = getSearchEngine().findAllByCriteria(criteria); PagedResult<Wikitty> result = resultId.cast(getWikittyStorage()); return result; } @@ -355,7 +355,7 @@ Map<TreeNode, Integer> result = new LinkedHashMap<TreeNode, Integer>(); - Map<String, Integer> search = getSearchEngin().findAllChildren(w); + Map<String, Integer> search = getSearchEngine().findAllChildren(w); Set<Entry<String, Integer>> children = search.entrySet(); for (Entry<String, Integer> child : children) { Integer count = child.getValue(); @@ -678,4 +678,14 @@ } } + /** + * Changes the data directory + * @param newDataDir the new data directory path + * @param oldDataDir the old data directory path. + * If null, the data in the old directory will not be copied. + */ + public void changeDataDir(String newDataDir, String oldDataDir) { + getSearchEngine().changeDataDir(newDataDir, oldDataDir); + } + } Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyProxy.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyProxy.java 2009-11-06 18:01:21 UTC (rev 315) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyProxy.java 2009-11-10 16:30:50 UTC (rev 316) @@ -410,5 +410,7 @@ wikittyService.freeJobResource(jobId); } - + public void changeDataDir(String newDataDir, String oldDataDir) { + wikittyService.changeDataDir(newDataDir, oldDataDir); + } } Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyService.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyService.java 2009-11-06 18:01:21 UTC (rev 315) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyService.java 2009-11-10 16:30:50 UTC (rev 316) @@ -264,4 +264,12 @@ */ void freeJobResource(String jobId); + /** + * Changes the data directory + * @param newDataDir the new data directory path + * @param oldDataDir the old data directory path. + * If null, the data in the old directory will not be copied. + */ + public void changeDataDir(String newDataDir, String oldDataDir); + } Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyServiceInMemory.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyServiceInMemory.java 2009-11-06 18:01:21 UTC (rev 315) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyServiceInMemory.java 2009-11-10 16:30:50 UTC (rev 316) @@ -37,10 +37,8 @@ import org.apache.commons.logging.LogFactory; import org.sharengo.wikitty.FieldType.TYPE; import org.sharengo.wikitty.search.And; -import org.sharengo.wikitty.search.Between; import org.sharengo.wikitty.search.BinaryOperator; import org.sharengo.wikitty.search.Element; -import org.sharengo.wikitty.search.Equals; import org.sharengo.wikitty.search.Restriction; /** @@ -221,11 +219,11 @@ } } - class WikittySearchEnginInMemory implements WikittySearchEngin { + class WikittySearchEngineInMemory implements WikittySearchEngine { WikittyStorageInMemory wikittyStorage; - public WikittySearchEnginInMemory(WikittyStorageInMemory wikittyStorage) { + public WikittySearchEngineInMemory(WikittyStorageInMemory wikittyStorage) { this.wikittyStorage = wikittyStorage; } @@ -233,6 +231,11 @@ // do nothing } + @Override + public void changeDataDir(String newDataDir, String oldDataDir) { + //Do nothing + } + public List<Command> prepare(WikittyTransaction transaction, Collection<Wikitty> wikitties) { // do nothing @@ -357,12 +360,12 @@ protected WikittyStorage wikittyStorage; protected WikittyExtensionStorage extensionStorage; - protected WikittySearchEngin searchEngin; + protected WikittySearchEngine searchEngine; public WikittyServiceInMemory() { extensionStorage = new WikittyExtensionStorageInMemory(); wikittyStorage = new WikittyStorageInMemory(); - searchEngin = new WikittySearchEnginInMemory((WikittyStorageInMemory)wikittyStorage); + searchEngine = new WikittySearchEngineInMemory((WikittyStorageInMemory)wikittyStorage); } public WikittyServiceInMemory(File persitenceFile) { @@ -392,8 +395,8 @@ } @Override - protected WikittySearchEngin getSearchEngin() { - return searchEngin; + protected WikittySearchEngine getSearchEngine() { + return searchEngine; } @Override Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/test/resources/log4j.properties =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/test/resources/log4j.properties 2009-11-06 18:01:21 UTC (rev 315) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/test/resources/log4j.properties 2009-11-10 16:30:50 UTC (rev 316) @@ -6,3 +6,4 @@ # Configuration by components log4j.rootLogger=ERROR, logConsole log4j.category.org.sharengo.wikitty=DEBUG +log4j.category.org.apache.solr=DEBUG

1 0

r315 - wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty
by kmorin＠users.nuiton.org 06 Nov '09

06 Nov '09

Author: kmorin Date: 2009-11-06 19:01:21 +0100 (Fri, 06 Nov 2009) New Revision: 315 Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyExtension.java Log: add addExtension method in the business entity Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java 2009-11-06 18:00:58 UTC (rev 314) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java 2009-11-06 18:01:21 UTC (rev 315) @@ -43,6 +43,12 @@ * Return list of all extensions for this object * @return */ + public void addExtension(WikittyExtension extension); + + /** + * Return list of all extensions for this object + * @return + */ public Collection<WikittyExtension> getExtensions(); /** Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java 2009-11-06 18:00:58 UTC (rev 314) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java 2009-11-06 18:01:21 UTC (rev 315) @@ -63,6 +63,11 @@ } @Override + public void addExtension(WikittyExtension extension) { + extensions.put(extension.getName(), extension); + } + + @Override public Collection<WikittyExtension> getExtensions() { return extensions.values(); } Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java 2009-11-06 18:00:58 UTC (rev 314) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java 2009-11-06 18:01:21 UTC (rev 315) @@ -63,6 +63,11 @@ } @Override + public void addExtension(WikittyExtension extension) { + getWikitty().addExtension(extension); + } + + @Override public Collection<WikittyExtension> getExtensions() { return getWikitty().getExtensions(); } Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyExtension.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyExtension.java 2009-11-06 18:00:58 UTC (rev 314) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/WikittyExtension.java 2009-11-06 18:01:21 UTC (rev 315) @@ -19,8 +19,10 @@ import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -130,8 +132,8 @@ return fields.get(fieldName); } - public Collection<String> getFieldNames() { - Collection<String> result = fields.keySet(); + public List<String> getFieldNames() { + List<String> result = new ArrayList<String>(fields.keySet()); return result; }

1 0

r314 - wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc
by kmorin＠users.nuiton.org 06 Nov '09

06 Nov '09

Author: kmorin Date: 2009-11-06 19:00:58 +0100 (Fri, 06 Nov 2009) New Revision: 314 Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyExtensionStorageJDBC.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyJDBCUtil.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyStorageJDBC.java Log: add order in the extension fields Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyExtensionStorageJDBC.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyExtensionStorageJDBC.java 2009-11-04 16:04:53 UTC (rev 313) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyExtensionStorageJDBC.java 2009-11-06 18:00:58 UTC (rev 314) @@ -58,6 +58,10 @@ */ static protected String selectionQ = "jdbc.queries.select"; /** + * ordered selection with where clause query property name + */ + static protected String orderedWhereSelectionQ = "jdbc.queries.select.where.order"; + /** * basic selection with where clause query property name */ static protected String whereSelectionQ = "jdbc.queries.select.where"; @@ -164,10 +168,11 @@ batch.append(String.format(conf.getProperty(adminInsertQ), ext.getId(), ext.getName(), ext.getVersion(), ext.getRequires() != null ? "'" + ext.getRequires() + "'" : "null")); - for (String fieldName : ext.getFieldNames()) { - FieldType type = ext.getFieldType(fieldName); + for (int i = 0 ; i < ext.getFieldNames().size() ; i++) { + FieldType type = ext.getFieldType(ext.getFieldNames().get(i)); batch.append(String.format(conf.getProperty(dataInsertQ), - ext.getId(), fieldName.replace(" ", WORD_SEPARATOR), type.toDefinition(fieldName.replace(" ", WORD_SEPARATOR)))); + ext.getId(), ext.getFieldNames().get(i).replace(" ", WORD_SEPARATOR), + type.toDefinition(ext.getFieldNames().get(i).replace(" ", WORD_SEPARATOR)), i)); } } else { log.warn("The extension is found " + id); @@ -353,8 +358,8 @@ LinkedHashMap<String, FieldType> fieldTypes = new LinkedHashMap<String, FieldType>(); //get the data with the id "id" in the data table ResultSet dataResultSet = statement.executeQuery( - String.format(conf.getProperty(whereSelectionQ), "*", - dataTable, COL_ID, "'" + id + "'")); + String.format(conf.getProperty(orderedWhereSelectionQ), "*", + dataTable, COL_ID, "'" + id + "'", COL_FIELDRANK)); dataResultSet.beforeFirst(); while (dataResultSet.next()) { String fieldDef = dataResultSet.getString(COL_FIELDTYPE); Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyJDBCUtil.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyJDBCUtil.java 2009-11-04 16:04:53 UTC (rev 313) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyJDBCUtil.java 2009-11-06 18:00:58 UTC (rev 314) @@ -71,6 +71,10 @@ */ static final public String COL_FIELDTYPE = "fieldType"; /** + * field rank in the extension_data table + */ + static final public String COL_FIELDRANK = "fieldRank"; + /** * boolean value column in the wikitty_data table */ static final public String COL_BOOLEAN_VALUE = "booleanValue"; Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyStorageJDBC.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyStorageJDBC.java 2009-11-04 16:04:53 UTC (rev 313) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyStorageJDBC.java 2009-11-06 18:00:58 UTC (rev 314) @@ -258,7 +258,7 @@ COL_TEXT_VALUE, wikitty.getId(), ext.getName(), fieldName + "[" + i + "/" + list.size() + "]", - "'" + value + "'")); + "'" + value.replace("'", "''") + "'")); } break; case WIKITTY: @@ -280,7 +280,7 @@ COL_TEXT_VALUE, wikitty.getId(), ext.getName(), fieldName + "[" + i + "/" + list.size() + "]", - "'" + value + "'")); + "'" + value.replace("'", "''") + "'")); } break; } @@ -344,7 +344,7 @@ batch.append(String.format( conf.getProperty(dataInsertQ), COL_TEXT_VALUE, wikitty.getId(), ext.getName(), - fieldName, "'" + stringValue + "'")); + fieldName, "'" + stringValue.replace("'", "''") + "'")); } else { System.out.println("#####DEBUG####" + type.toDefinition(fieldName)); if (type.isNotNull()) { @@ -359,7 +359,7 @@ batch.append(String.format( conf.getProperty(dataInsertQ), COL_TEXT_VALUE, wikitty.getId(), ext.getName(), - fieldName, "'" + objectValue.toString() + "'")); + fieldName, "'" + objectValue.toString().replace("'", "''") + "'")); } else { System.out.println("#####DEBUG####" + type.toDefinition(fieldName)); if (type.isNotNull()) { @@ -374,7 +374,7 @@ batch.append(String.format( conf.getProperty(dataInsertQ), COL_TEXT_VALUE, wikitty.getId(), ext.getName(), - fieldName, "'" + objectValue.toString() + "'")); + fieldName, "'" + objectValue.toString().replace("'", "''") + "'")); } else { System.out.println("#####DEBUG####" + type.toDefinition(fieldName)); if (type.isNotNull()) { @@ -579,7 +579,7 @@ * @param id the id of the wikitty to restore * @param version the version of the wikitty to restore * @param extensionList the list of the extensions of the wikitty to restore - * @param resulSet the ResultSet as the result of the selection of the in the data table + * @param resultSet the ResultSet as the result of the selection of the in the data table * @param fqFieldName minimum field to restore * @return */

1 0

r313 - wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr
by kmorin＠users.nuiton.org 04 Nov '09

04 Nov '09

Author: kmorin Date: 2009-11-04 17:04:53 +0100 (Wed, 04 Nov 2009) New Revision: 313 Modified: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java Log: Add getExtensions method Modified: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java 2009-11-04 16:03:24 UTC (rev 312) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java 2009-11-04 16:04:53 UTC (rev 313) @@ -38,6 +38,7 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; import org.sharengo.wikitty.Criteria; import org.sharengo.wikitty.FacetTopic; import org.sharengo.wikitty.FieldType; @@ -202,7 +203,7 @@ } /** solr server */ - protected SolrServer solrServer; + protected SolrServer solrServer; /** Criteria transformer to solr */ protected Restriction2Solr restriction2Solr; /** Field modifier use to transform to solr format */ @@ -214,7 +215,6 @@ CoreContainer.Initializer initializer = new CoreContainer.Initializer(); CoreContainer coreContainer = initializer.initialize(); solrServer = new EmbeddedSolrServer(coreContainer, ""); - fieldModifier = new TypeFieldModifer(extensionStorage); restriction2Solr = new Restriction2Solr(fieldModifier); @@ -462,7 +462,7 @@ /** * Create all index document to used to modify indexation. * this method don't modify index - * @param wikitties all wikitties object to index + * @param w wikitty object to index * @return list of SolrInputDocument used to modify index */ protected SolrInputDocument createIndexDocument(Wikitty w) {

1 0

r312 - wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty
by kmorin＠users.nuiton.org 04 Nov '09

04 Nov '09

Author: kmorin Date: 2009-11-04 17:03:24 +0100 (Wed, 04 Nov 2009) New Revision: 312 Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java Log: Add getExtensions method Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java 2009-11-04 10:20:45 UTC (rev 311) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntity.java 2009-11-04 16:03:24 UTC (rev 312) @@ -40,9 +40,15 @@ public void setWikittyVersion(String version); /** - * Return list of all extension for this object + * Return list of all extensions for this object * @return */ + public Collection<WikittyExtension> getExtensions(); + + /** + * Return list of all extension names for this object + * @return + */ public Collection<String> getExtensionNames(); /** Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java 2009-11-04 10:20:45 UTC (rev 311) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityBean.java 2009-11-04 16:03:24 UTC (rev 312) @@ -61,8 +61,13 @@ public void setWikittyVersion(String version) { this.version = version; } - + @Override + public Collection<WikittyExtension> getExtensions() { + return extensions.values(); + } + + public Collection<String> getExtensionNames() { Collection<String> result = extensions.keySet(); return result; Modified: wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java 2009-11-04 10:20:45 UTC (rev 311) +++ wikengo_core-wikitty/wikengo_core-wikitty-api/src/main/java/org/sharengo/wikitty/BusinessEntityWikitty.java 2009-11-04 16:03:24 UTC (rev 312) @@ -62,6 +62,11 @@ getWikitty().setVersion(version); } + @Override + public Collection<WikittyExtension> getExtensions() { + return getWikitty().getExtensions(); + } + public void setWikitty(Wikitty wikitty) { wikitty = upgradeData(wikitty, getStaticExtensions()); for ( WikittyExtension ext : getStaticExtensions() ) { @@ -90,7 +95,32 @@ } public Object getField(String ext, String fieldName) { - Object result = getWikitty().getFieldAsObject(ext, fieldName); + Object result = null; + FieldType type = getFieldType(ext, fieldName); + if(type.isCollection()) { + result = getWikitty().getFieldAsObject(ext, fieldName); + } else { + switch(type.getType()) { + case BOOLEAN: + result = getWikitty().getFieldAsBoolean(ext, fieldName); + break; + case DATE: + result = getWikitty().getFieldAsDate(ext, fieldName); + break; + case NUMERIC: + result = getWikitty().getFieldAsBigDecimal(ext, fieldName); + break; + case STRING: + result = getWikitty().getFieldAsString(ext, fieldName); + break; + case WIKITTY: + result = getWikitty().getFieldAsString(ext, fieldName); + break; + default: + result = getWikitty().getFieldAsObject(ext, fieldName); + break; + } + } return result; }

1 0

04 Nov '09

Author: kmorin Date: 2009-11-04 11:20:45 +0100 (Wed, 04 Nov 2009) New Revision: 311 Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/protwords.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/spellwords.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/synonyms.txt wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/wikitty/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/wikitty/solr/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/java/org/sharengo/wikitty/solr/test/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/META-INF/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/META-INF/spring/ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties Log: Separation of solr Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/core-wikitty-solr-impl.iml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" inherit-compiler-output="false"> + <output url="file://$MODULE_DIR$/target/classes" /> + <output-test url="file://$MODULE_DIR$/target/test-classes" /> + <content url="file://$MODULE_DIR$"> + <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" /> + <sourceFolder url="file://$MODULE_DIR$/src/main/resources" isTestSource="false" /> + <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" /> + <sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" /> + <excludeFolder url="file://$MODULE_DIR$/target" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + <orderEntry type="module" module-name="core-wikitty-api" exported="" /> + <orderEntry type="library" exported="" scope="RUNTIME" name="Maven: log4j:log4j:1.2.14" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-logging:commons-logging:1.1.1" level="project" /> + <orderEntry type="library" exported="" name="Maven: xpp3:xpp3:1.1.4c" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-core:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-common:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-solrj:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-codec:commons-codec:1.3" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.commons:commons-io:1.3.2" level="project" /> + <orderEntry type="library" exported="" name="Maven: commons-fileupload:commons-fileupload:1.2" level="project" /> + <orderEntry type="library" exported="" name="Maven: stax:stax:1.2.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: stax:stax-api:1.0.1" level="project" /> + <orderEntry type="library" exported="" name="Maven: net.java.dev.stax-utils:stax-utils:20040917" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-analyzers:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-core:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-highlighter:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-queries:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-snowball:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-lucene-spellchecker:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.solr:solr-commons-csv:1.3.0" level="project" /> + <orderEntry type="library" exported="" name="Maven: woodstox:wstx-asl:3.2.7" level="project" /> + <orderEntry type="library" exported="" name="Maven: org.apache.geronimo.specs:geronimo-stax-api_1.0_spec:1.0.1" level="project" /> + <orderEntry type="library" exported="" scope="RUNTIME" name="Maven: javax.servlet:servlet-api:2.5" level="project" /> + </component> +</module> + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,92 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + + <modelVersion>4.0.0</modelVersion> + +  +  +  + + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty-solr-impl</artifactId> + + <parent> + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty</artifactId> + <version>1.0-CL-SNAPSHOT</version> + </parent> + + <dependencies> +  + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>core-wikitty-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + +  + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-core</artifactId> + <version>1.3.0</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.5</version> + <scope>runtime</scope> + </dependency> + + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + </dependencies> + +  +  +  + + <name>org.sharengo.wikengo.core-wikitty-solr-impl</name> + + <description>hbase impl of wikitty</description> + <inceptionYear>2009</inceptionYear> + +  +  +  + + <packaging>jar</packaging> + + <properties> + <maven.test.skip>true</maven.test.skip> + <maven.test.failure.ignore>true</maven.test.failure.ignore> + </properties> + +  +  +  + + <repositories> + <repository> + <id>sharengo-maven2-repository</id> + <name>Sharengo's repository</name> + <url>http://maven2.sharengo.org</url> + <snapshots> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </snapshots> + <releases> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </releases> + </repository> + </repositories> + +</project> + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/pom.xml~ 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,92 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + + <modelVersion>4.0.0</modelVersion> + +  +  +  + + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty-solr-impl</artifactId> + + <parent> + <groupId>org.sharengo.wikengo</groupId> + <artifactId>core-wikitty</artifactId> + <version>1.0-CL-SNAPSHOT</version> + </parent> + + <dependencies> +  + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>core-wikitty-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + +  + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-core</artifactId> + <version>1.3.0</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.5</version> + <scope>runtime</scope> + </dependency> + + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + </dependencies> + +  +  +  + + <name>org.sharengo.wikengo.core-wikitty-solr-impl</name> + + <description>hbase impl of wikitty</description> + <inceptionYear>2009</inceptionYear> + +  +  +  + + <packaging>jar</packaging> + + <properties> + <maven.test.skip>true</maven.test.skip> + <maven.test.failure.ignore>true</maven.test.failure.ignore> + </properties> + +  +  +  + + <repositories> + <repository> + <id>sharengo-maven2-repository</id> + <name>Sharengo's repository</name> + <url>http://maven2.sharengo.org</url> + <snapshots> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </snapshots> + <releases> + <enabled>true</enabled> + <checksumPolicy>warn</checksumPolicy> + </releases> + </repository> + </repositories> + +</project> + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/Restriction2Solr.java 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,397 @@ +package org.sharengo.wikitty.solr; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.sharengo.wikitty.WikittyException; +import org.sharengo.wikitty.search.And; +import org.sharengo.wikitty.search.AssociatedRestriction; +import org.sharengo.wikitty.search.Between; +import org.sharengo.wikitty.search.Contains; +import org.sharengo.wikitty.search.Element; +import org.sharengo.wikitty.search.EndsWith; +import org.sharengo.wikitty.search.Equals; +import org.sharengo.wikitty.search.Greater; +import org.sharengo.wikitty.search.GreaterOrEqual; +import org.sharengo.wikitty.search.In; +import org.sharengo.wikitty.search.Keyword; +import org.sharengo.wikitty.search.Less; +import org.sharengo.wikitty.search.LessOrEqual; +import org.sharengo.wikitty.search.Not; +import org.sharengo.wikitty.search.NotEquals; +import org.sharengo.wikitty.search.Or; +import org.sharengo.wikitty.search.Restriction; +import org.sharengo.wikitty.search.RestrictionHelper; +import org.sharengo.wikitty.search.StartsWith; + +/** + * @author "Nicolas Chapurlat" <nicolas.chapurlat(a)logica.com> + * @author "Guillaume Dufrêne" <dufrene(a)argia.fr> + * + * This class is used to parse Restriction to create lucene request on + * content. Every operators describe in RestrictionName is handle. Parsing may + * throw exception when restriction parameters are incorrect. + */ +public class Restriction2Solr { + + private static final int MAX_SUBQUERY_RESULT = 100; + + /** use to permit client to modify fieldname during query generation */ + static public interface FieldModifier { + public String convertToSolr(String fieldname); + public String convertToField(String solrName); + } + final static protected FieldModifier dummyFieldModifier = new FieldModifier() { + public String convertToSolr(String fieldname) { + return fieldname; + } + public String convertToField(String solrName) { + return solrName; + } + }; + + protected FieldModifier fieldModifer; + + public Restriction2Solr() { + this(dummyFieldModifier); + } + + public Restriction2Solr(FieldModifier fieldModifer) { + this.fieldModifer = fieldModifer; + } + + public String toSolr(Restriction restriction) { + return toSolr(restriction, null); + } + + public String toSolr(Restriction restriction, SolrServer solr) + throws WikittyException { + // ParameterValidator.checkNullParameter(restriction, "restriction"); + switch (restriction.getName()) { + case TRUE: + return true2solr(); + case FALSE: + return false2solr(); + case NOT: + Not not = (Not) restriction; + return not2solr(not); + case AND: + And and = (And) restriction; + return and2solr(and); + case OR: + Or or = (Or) restriction; + return or2solr(or); + case EQUALS: + Equals eq = (Equals) restriction; + return eq2solr(eq); + case NOT_EQUALS: + NotEquals neq = (NotEquals) restriction; + return neq2solr(neq); + case LESS: + Less less = (Less) restriction; + return less2solr(less); + case LESS_OR_EQUAL: + LessOrEqual lessEq = (LessOrEqual) restriction; + return lessEq2solr(lessEq); + case GREATER: + Greater great = (Greater) restriction; + return great2solr(great); + case GREATER_OR_EQUAL: + GreaterOrEqual greatEq = (GreaterOrEqual) restriction; + return greatEq2solr(greatEq); + case BETWEEN: + Between between = (Between) restriction; + return between2solr(between); + case CONTAINS: + Contains contains = (Contains) restriction; + return contains2solr(contains); + case IN: + In in = (In) restriction; + return in2solr(in); + case STARTS_WITH: + StartsWith start = (StartsWith) restriction; + return start2solr(start); + case ENDS_WITH: + EndsWith end = (EndsWith) restriction; + return end2solr(end); + case ASSOCIATED: + AssociatedRestriction associated = (AssociatedRestriction) restriction; + return associated2solr(associated, solr); + case KEYWORD: + Keyword keyword = (Keyword) restriction; + return keyword2solr(keyword); + default: + throw new WikittyException("this kind of restriction is not supported : " + + restriction.getName().toString()); + } + } + + private String in2solr(In in) { + boolean first = true; + String result = in.getElement().getName() + ":["; + for( String value : in.getValue() ) { + if ( !first ) { result += ", "; first = false; } + result += value; + } + result +="]"; + return result; + } + + private String associated2solr(AssociatedRestriction associated, SolrServer solr) throws WikittyException { + String subQuery = toSolr( associated.getRestriction() ); + SolrQuery query = new SolrQuery(subQuery); + query.setRows(MAX_SUBQUERY_RESULT); + QueryResponse resp = null; + try { + resp = solr.query(query); + } catch (SolrServerException e) { + throw new WikittyException("Unable to execute associative query on " + associated.getElement().getName(), e); + } + SolrDocumentList solrResults = resp.getResults(); + + Restriction generatedRestriction = null; + long size = solrResults.size(); + if ( size == 0 ) { + throw new WikittyException("Associated " + associated.getElement().getName() + " do not retrieved any result"); + } + if ( size == 1 ) { + generatedRestriction = RestrictionHelper.eq( associated.getElement(), (String) solrResults.get(0).getFieldValue("id") ); + } else { + List<String> ids = new ArrayList<String>(solrResults.size()); + for (SolrDocument doc : solrResults) { + String id = (String) doc.getFieldValue("id"); // FIXME : set a constant field reference + ids.add(id); + } + generatedRestriction = new In(associated.getElement(), ids); + } + Restriction parent = associated.getParentRestrictionDto(); + And and; + if ( parent instanceof And ) { + and = (And) parent; + and.getRestrictions().add( generatedRestriction ); + } else { + and = RestrictionHelper.and( Arrays.asList(new Restriction[]{ associated.getParentRestrictionDto(), generatedRestriction }) ); + } + return toSolr(and); + } + + private String not2solr(Not not) throws WikittyException { + if (not.getRestriction() == null) { + throw new WikittyException( "not.restriction" ); + } + return "( *:* - " + toSolr(not.getRestriction()) + " )"; + } + + private String and2solr(And and) throws WikittyException { + if (and.getRestrictions() == null) { + throw new WikittyException( "and.restrictions is null" ); + } + if (and.getRestrictions().size() < 2) { + throw new WikittyException( "AND is an operator that handle 2 operand at least"); + } + boolean first = true; + StringBuffer result = new StringBuffer(); + for (Restriction restriction : and.getRestrictions()) { + if (first) { + result.append("( ").append(toSolr(restriction)); + first = false; + } else { + result.append(" AND ").append(toSolr(restriction)); + } + } + return result.append(" )").toString(); + } + + private String or2solr(Or or) throws WikittyException { + if (or.getRestrictions() == null) { + throw new WikittyException("or.restrictions is null"); + } + if (or.getRestrictions().size() < 2) { + throw new WikittyException("OR is an operator that handle 2 operand at least"); + } + boolean first = true; + StringBuffer result = new StringBuffer(); + for (Restriction restriction : or.getRestrictions()) { + if (first) { + result.append("( "); + first = false; + } else { + result.append(" OR "); + } + result.append(toSolr(restriction)); + } + return result.append(" )").toString(); + } + + private String eq2solr(Equals eq) throws WikittyException { + return element2solr(eq.getElement()) + ":" + value2solr(eq.getValue()); + } + + private String neq2solr(NotEquals neq) + throws WikittyException { + return "( *:* - " + element2solr(neq.getElement()) + ":" + + value2solr(neq.getValue()) + " )"; + } + + private String less2solr(Less less) throws WikittyException { + return element2solr(less.getElement()) + ":{* TO " + + value2solr(less.getValue()) + "}"; + } + + private String lessEq2solr(LessOrEqual lessEq) + throws WikittyException { + return element2solr(lessEq.getElement()) + ":[* TO " + + value2solr(lessEq.getValue()) + "]"; + } + + private String great2solr(Greater great) + throws WikittyException { + return element2solr(great.getElement()) + ":{" + + value2solr(great.getValue()) + " TO *}"; + } + + private String greatEq2solr(GreaterOrEqual greatEq) + throws WikittyException { + return element2solr(greatEq.getElement()) + ":[" + + value2solr(greatEq.getValue()) + " TO *]"; + } + + private String between2solr(Between between) + throws WikittyException { + if (between.getElement() == null) { + throw new WikittyException("contains.element"); + } + if (between.getMin() == null) { + throw new WikittyException("contains.min"); + } + if (between.getMax() == null) { + throw new WikittyException("contains.max"); + } + return element2solr(between.getElement()) + ":[" + + value2solr(between.getMin()) + " TO " + + value2solr(between.getMax()) + "]"; + } + + private String contains2solr(Contains contains) + throws WikittyException { + if (contains.getElement() == null) { + throw new WikittyException("contains.element"); + } + if (contains.getValue() == null) { + throw new WikittyException("contains.values"); + } + if (contains.getValue().size() < 1) { + throw new WikittyException("CONTAINS is an operator that handle 1 operand at least"); + } + boolean first = true; + StringBuffer result = new StringBuffer(); + for (String value : contains.getValue()) { + if (first) { + result.append("( "); + first = false; + } else { + result.append(" AND "); + } + result.append("( "); + result.append(element2solr(contains.getElement())) + .append(":").append(value2solr(value)); + result.append(" OR "); + result.append(element2solr(contains.getElement())) + .append(":").append(value2solr(value)); + result.append(" OR "); + result.append(element2solr(contains.getElement())) + .append(":").append(value2solr(value)); + result.append(" )"); + } + return result.append(" )").toString(); + } + + private String start2solr(StartsWith start) + throws WikittyException { + return element2solr(start.getElement()) + ":" + + value2solr(start.getValue(), true, false); + } + + private String end2solr(EndsWith end) { + return element2solr(end.getElement()) + ":" + value2solr(end.getValue()); + } + + private String true2solr() { + return "( *:* )"; + } + + private String false2solr() { + return "( *:* - *:* )"; + } + + private String keyword2solr(Keyword keyword) { + return value2solr(keyword.getValue()); + } + + private String value2solr(String value) { + return value2solr(value, false, false); + } + + private String value2solr(String value, boolean starOnEnd, + boolean revert) { + String result; + if (value != null) { + if (value.equals("*")) { + return "*"; + } + + if (revert) { + result = Restriction2Solr.escapeValue(reverse(value)); + } else { + result = Restriction2Solr.escapeValue(value); + // result = value; + } + } else { + result = ""; + } + if (starOnEnd) { + result += "*"; + } + if (result.contains(" ")) { + result = "\"" + result + "\""; + } + return result; + } + + private String element2solr(Element element) throws WikittyException { + String result = element.getName(); + if(Element.ELT_EXTENSION.equals(result)) { + result = WikittySearchEngineSolr.SOLR_EXTENSIONS; + } else if(Element.ELT_ID.equals(result)) { + result = WikittySearchEngineSolr.SOLR_ID; + } else { + result = element.getName(); + } + result = fieldModifer.convertToSolr(result); + return result; + } + + private static String escapeValue(String value) { + final String LUCENE_REPLACE_PATTERN = "\\+" + "|-" + "|&&" + "|\\|\\|" + + "|!" + "|\$|\$" + "|\\[|\\]" + "|\\{|\\}" + "|\\^" + "|\"" + + "|\\~" + "|\\*" + "|\\?" + "|:" + "|\\\\"; + return value.replaceAll(LUCENE_REPLACE_PATTERN, "\\\\$0"); + } + + private String reverse(String source) { + int i, len = source.length(); + StringBuffer dest = new StringBuffer(len); + + for (i = (len - 1); i >= 0; i--) + dest.append(source.charAt(i)); + return dest.toString(); + } +} + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittyQueryParser.java 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,70 @@ +package org.sharengo.wikitty.solr; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Query; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.LuceneQParserPlugin; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SolrQueryParser; + +/** + * Add allow leading wildcard + * setAllowLeadingWildcard(true); + * setLowercaseExpandedTerms(true); + * Example: <code>{!wikitty q.op=AND df=text sort='price asc'}myfield:foo +bar -baz</code> + * More information @see LuceneQParserPlugin + */ +public class WikittyQueryParser extends LuceneQParserPlugin { + + public static String NAME = "wikitty"; + + @Override + public void init(NamedList args) { + } + + @Override + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new SolrQParser(qstr, localParams, params, req); + } +} + +class SolrQParser extends QParser { + + static private Log log = LogFactory.getLog(SolrQParser.class); + String sortStr; + SolrQueryParser lparser; + + public SolrQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + @Override + public Query parse() throws ParseException { + + String defaultField = getParam(CommonParams.DF); + lparser = new SolrQueryParser(this, defaultField); + lparser.setAllowLeadingWildcard(true); + lparser.setLowercaseExpandedTerms(true); + + String opParam = getParam(QueryParsing.OP); + if (opParam != null) { + lparser.setDefaultOperator("AND".equals(opParam) ? QueryParser.Operator.AND : QueryParser.Operator.OR); + } + + String qstr = getString(); + log.debug("Query parse : " + qstr); + return lparser.parse(qstr); + } + + @Override + public String[] getDefaultHighlightFields() { + return new String[]{lparser.getField()}; + } +} Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/java/org/sharengo/wikitty/solr/WikittySearchEngineSolr.java 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,652 @@ +/* *##% + * Copyright (c) 2009 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.sharengo.wikitty.solr; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.core.CoreContainer; +import org.sharengo.wikitty.Criteria; +import org.sharengo.wikitty.FacetTopic; +import org.sharengo.wikitty.FieldType; +import org.sharengo.wikitty.FieldType.TYPE; +import org.sharengo.wikitty.PagedResult; +import org.sharengo.wikitty.TreeNode; +import org.sharengo.wikitty.UpdateResponse; +import org.sharengo.wikitty.Wikitty; +import org.sharengo.wikitty.WikittyException; +import org.sharengo.wikitty.WikittyExtension; +import org.sharengo.wikitty.WikittyExtensionStorage; +import org.sharengo.wikitty.WikittySearchEngin; +import org.sharengo.wikitty.WikittyTransaction; +import org.sharengo.wikitty.search.Search; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittySearchEngineSolr implements WikittySearchEngin { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittySearchEngineSolr.class); + + /** id field in solr */ + static final protected String SOLR_ID = "id"; + + /** extensions field name in solr */ + static final public String SOLR_EXTENSIONS = "extensions"; + + /** extension use to store field without extension to search on all extesnion */ + static final public String SOLR_ALL_EXTENSIONS = "all"; + + /** Precise the query parser to use, is allow leading wildcard */ + static final public String SOLR_QUERY_PARSER = "{!wikitty}"; + + // Use for indexation tree node + static final public String TREENODE_PREFIX = TreeNode.EXT_TREENODE + "."; + static final public String TREENODE_CHILD = TREENODE_PREFIX + "wikittyId"; + static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; + static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; + + static protected class CommandSolr implements Command { + /** if is store command toStore is true, if is delete toStore is false */ + boolean toStore; + String id; + Wikitty wikitty; + /** use to store command */ + public CommandSolr(Wikitty wikitty) { + this.wikitty = wikitty; + this.toStore = true; + } + /** use for deletion command */ + public CommandSolr(String id) { + this.id = id; + this.toStore = false; + } + } + + static protected class TypeFieldModifer implements Restriction2Solr.FieldModifier { + protected WikittyExtensionStorage extensionStorage; + public TypeFieldModifer(WikittyExtensionStorage extensionStorage) { + this.extensionStorage = extensionStorage; + } + + public String convertToSolr(String fqfieldname) { + String result = fqfieldname; + String[] searchField = fqfieldname.split("\\."); + + if (searchField.length == 2) { + String extName = searchField[0]; + String fieldName = searchField[1]; + + String version = extensionStorage.getLastVersion(extName); + if (version != null) { // not valid extension is version == null + WikittyExtension ext = extensionStorage.restore( + WikittyExtension.computeId(extName, version)); + FieldType type = ext.getFieldType(fieldName); + if (type != null) { // type can be null if extension version differ + result = WikittySearchEngineSolr.getSolrFieldName( + fqfieldname, type); + } else { + log.debug(String.format( + "Field '%s' could not be found in '%s'", + fqfieldname, extName)); + } + } else { + log.warn("No version for extension " + extName); + } + } + + if (searchField.length == 3) { + String extName = searchField[0]; + String fieldName = searchField[1]; + String fieldType = searchField[2]; + + if(Criteria.ALL_EXTENSIONS.equals(extName)) { + TYPE type = FieldType.TYPE.valueOf(fieldType); + String solrFieldName = SOLR_ALL_EXTENSIONS + "." + fieldName; + result = WikittySearchEngineSolr.getSolrFieldName(solrFieldName, type); + } else { + log.warn("All extension not found " + extName); + } + } + + return result; + } + + public String convertToField(String solrName) { + String fieldname = solrName.replaceAll("(_b$)|(_dt$)|(_t$)|(_l$)", ""); + return fieldname; + } + + } + + /** + * Store parent for each node and search in index if not known parent + */ + protected static class ParentMap extends HashMap<String, String> { + + protected WikittySearchEngineSolr searchEngineSolr; + + public ParentMap(WikittySearchEngineSolr searchEngineSolr) { + this.searchEngineSolr = searchEngineSolr; + } + + @Override + public String get(Object key) { + String id = String.valueOf(key); + String parent = super.get(id); + + // If not found in map, search in index + if(parent == null) { + SolrDocument doc = searchEngineSolr.findById(id); + if(doc == null) { + // is root + return null; + } + parent = (String) doc.getFieldValue(TreeNode.FQ_FIELD_PARENT); + put(id, parent); + } + return parent; + } + + /** + * Add parent map in transaction if is not exists and return it + */ + public static ParentMap getParentMap( + WikittySearchEngineSolr searchEngineSolr, + WikittyTransaction transaction) { + ParentMap value = (ParentMap) transaction.getTagValue(ParentMap.class); + if(value == null) { + value = new ParentMap(searchEngineSolr); + transaction.setTagValue(ParentMap.class, value); + } + return value; + } + } + + /** solr server */ + protected SolrServer solrServer; + /** Criteria transformer to solr */ + protected Restriction2Solr restriction2Solr; + /** Field modifier use to transform to solr format */ + protected TypeFieldModifer fieldModifier; + + public WikittySearchEngineSolr(WikittyExtensionStorage extensionStorage) { + super(); + try { + CoreContainer.Initializer initializer = new CoreContainer.Initializer(); + CoreContainer coreContainer = initializer.initialize(); + solrServer = new EmbeddedSolrServer(coreContainer, ""); + + fieldModifier = new TypeFieldModifer(extensionStorage); + restriction2Solr = new Restriction2Solr(fieldModifier); + + } catch (Exception eee) { + throw new WikittyException(eee); + } + } + + public void clear() { + try { + solrServer.deleteByQuery("*:*"); + solrServer.commit(true, true); + } catch (Exception eee) { + throw new WikittyException(eee); + } + } + + public List<Command> prepare(WikittyTransaction transaction, + Collection<Wikitty> wikitties) { + List<Command> result = new ArrayList<Command>(wikitties.size()); + for (Wikitty w : wikitties) { + + // Store known parent in transaction + if(w.hasExtension(TreeNode.EXT_TREENODE)) { + ParentMap parentMap = ParentMap.getParentMap(this, transaction); + String wikittyId = w.getId(); + String parentId = w.getFieldAsWikitty( + TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); + parentMap.put(wikittyId, parentId); + } + + result.add(new CommandSolr(w)); + } + return result; + } + + public UpdateResponse commit(WikittyTransaction transaction, + List<Command> wikittyIndexationCommandList) { + try { + for (Command c : wikittyIndexationCommandList) { + CommandSolr cs = (CommandSolr) c; + if (cs.toStore) { + SolrInputDocument doc = createIndexDocument(cs.wikitty); + solrServer.add(doc); + + Collection<SolrInputDocument> docs = createTreeNodeIndex(transaction, cs.wikitty); + if(!docs.isEmpty()) { + solrServer.add(docs); + } + + } else { + // Delete all reference on tree + solrServer.deleteByQuery(TREENODE_CHILD + ":" + cs.id); + // FIXME: jru 20091027 delete node in middle in tree + solrServer.deleteByQuery(TREENODE_PREFIX + cs.id + ":" + TREENODE_EMPTY); + + solrServer.deleteById(cs.id); + } + } + solrServer.commit(); + + // no specific result needed + UpdateResponse result = new UpdateResponse(); + return result; + } catch (Exception eee) { + throw new WikittyException(eee); + } + } + + public List<Command> delete(WikittyTransaction transaction, + List<String> idList) throws WikittyException { + List<Command> result = new ArrayList<Command>(idList.size()); + for (String id : idList) { + result.add(new CommandSolr(id)); + } + return result; + } + + public PagedResult<String> findAllByCriteria(Criteria criteria) { + try { + + // Create query with restriction + String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); + + // Add pagged + int firstIndex = criteria.getFirstIndex(); + int endIndex = criteria.getEndIndex(); + + query.setStart(firstIndex); + int nbRows; + if (endIndex == -1) { + // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR + nbRows = Integer.MAX_VALUE - firstIndex; + } else { + nbRows = endIndex - firstIndex + 1; + } + query.setRows(nbRows); + + // Add sorting + List<String> sortAscending = criteria.getSortAscending(); + if(sortAscending != null) { + for (String sort : sortAscending) { + String tranform = fieldModifier.convertToSolr(sort); + query.setSortField(tranform, SolrQuery.ORDER.asc); + } + } + + List<String> sortDescending = criteria.getSortDescending(); + if(sortDescending != null) { + for (String sort : sortDescending) { + String tranform = fieldModifier.convertToSolr(sort); + query.setSortField(tranform, SolrQuery.ORDER.desc); + } + } + + // Add faceting + List<String> facetField = criteria.getFacetField(); + List<Criteria> facetCriteria = criteria.getFacetCriteria(); + + // use to map query string to criteria facet name + Map<String, String> facetQueryToName = new HashMap<String, String>(); + + if ((facetField != null && !facetField.isEmpty()) + || (facetCriteria != null && !facetCriteria.isEmpty())) { + query.setFacet(true); + query.setFacetMinCount(1); + // query.setFacetLimit(8); // no limit actualy + + // field facetisation + if (facetField != null) { + for (String fqfieldName : facetField) { + String tranform = fieldModifier.convertToSolr(fqfieldName); + query.addFacetField(tranform); + } + } + + // query facetisation + if (facetCriteria != null) { + for (Criteria facet : facetCriteria) { + String queryFacet = + restriction2Solr.toSolr(facet.getRestriction()); + facetQueryToName.put(queryFacet, facet.getName()); + query.addFacetQuery(queryFacet); + } + } + } + + QueryResponse resp = solrServer.query(query); + SolrDocumentList solrResults = resp.getResults(); + + Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); + if (facetField != null && !facetField.isEmpty()) { + for (FacetField facet : resp.getFacetFields()) { + String facetName = fieldModifier.convertToField(facet.getName()); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + if (facet.getValues() != null) { + for (FacetField.Count value : facet.getValues()) { + String topicName = value.getName(); + if(!topicName.endsWith(TREENODE_EMPTY)) { + int topicCount = (int) value.getCount(); + FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); + topics.add(topic); + } + } + } + facets.put(facetName, topics); + } + } + if (facetCriteria != null && !facetCriteria.isEmpty()) { + for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { + String facetName = facet.getKey(); + // don't use contains because, map can have key with null value + if (null != facetQueryToName.get(facetName)) { + facetName = facetQueryToName.get(facetName); + } + Integer count = facet.getValue(); + List<FacetTopic> topics = new ArrayList<FacetTopic>(); + FacetTopic topic = new FacetTopic(facetName, facetName, count); + topics.add(topic); + facets.put(facetName, topics); + } + } + + List<String> ids = new ArrayList<String>(solrResults.size()); + for (SolrDocument doc : solrResults) { + String id = (String) doc.getFieldValue(SOLR_ID); + ids.add(id); + } + + int numFound = (int)resp.getResults().getNumFound(); + PagedResult<String> result = new PagedResult<String>( + firstIndex, numFound, queryString, facets, ids); + + return result; + } catch (SolrServerException e) { + throw new WikittyException(e); + } + } + + public Map<String, Integer> findAllChildren(Wikitty w) { + String wikittyId = w.getId(); + + String parent = w.getFieldAsWikitty(TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); + if(parent == null) { + parent = TREENODE_ROOT; + } else { + parent = TREENODE_PREFIX + parent; + } + + // Find count with facet, if the node not contain recurcively content, + // the node not found with facet + Criteria criteria = Search.query().eq(parent, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(0) + .addFacetField(TREENODE_PREFIX + wikittyId); + PagedResult<String> search = findAllByCriteria(criteria); + + Map<String, Integer> counts = new HashMap<String, Integer>(); + List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); + if(topics != null) { + for (FacetTopic topic : topics) { + String topicName = topic.getTopicName(); + int topicCount = topic.getCount(); + counts.put(topicName, topicCount); + } + } + + log.debug("Facet result " + counts); + + // Find all children, add the other node not found with facet + criteria = Search.query().eq(TreeNode.FQ_FIELD_PARENT, wikittyId).criteria() + .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); + search = findAllByCriteria(criteria); + + List<String> children = search.getAll(); + for (String child : children) { + if(!counts.containsKey(child)) { + counts.put(child, 0); + } + } + + return counts; + } + + /** + * Create all index document to used to modify indexation. + * this method don't modify index + * @param wikitties all wikitties object to index + * @return list of SolrInputDocument used to modify index + */ + protected SolrInputDocument createIndexDocument(Wikitty w) { + log.debug("index wikitty " + w.getId()); + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField(SOLR_ID, w.getId()); + + for (String name : w.getExtensionNames()) { + doc.addField(SOLR_EXTENSIONS, name); + } + + for (String fqfieldName : w.fieldNames()) { + FieldType type = w.getFieldType(fqfieldName); + // do not index wikitty field reference +// if (type.getType() == FieldType.TYPE.WIKITTY) { +// continue; +// } + String solrFqFieldName = getSolrFieldName(fqfieldName, type ); + String[] extAndName = solrFqFieldName.split("\\."); + String solrFieldName = SOLR_ALL_EXTENSIONS + "." + extAndName[1]; + if (type.isCollection()) { + Collection collection = (Collection) w.getFqField(fqfieldName); + if (collection != null) { + for (Object objectValue : collection) { + if (objectValue != null) { + // String value = WikittyUtil.toString(type, objectValue); + Object value = objectValue; + doc.addField(solrFqFieldName, value); + doc.addField(solrFieldName, value); + } + } + } + } else { + Object objectValue = w.getFqField(fqfieldName); + if (objectValue != null) { + // String value = WikittyUtil.toString(type, objectValue); + Object value = objectValue; + log.debug("index field " + solrFqFieldName + " with value '" + value + "'"); + doc.addField(solrFqFieldName, value); + doc.addField(solrFieldName, value); + } + } + } + + return doc; + } + + /** + * Create a doc for each child for a tree node, his contains the path for + * the child. Update old doc when the node is moved and child is deleted. + */ + protected Collection<SolrInputDocument> createTreeNodeIndex( + WikittyTransaction transaction, Wikitty w) + throws IOException, SolrServerException { + + Collection<SolrInputDocument> result = new ArrayList<SolrInputDocument>(); + if(w.hasExtension(TreeNode.EXT_TREENODE)) { + String treeNodeId = w.getId(); + ParentMap parentMap = (ParentMap) transaction.getTagValue(ParentMap.class); + + // Init new children with all children in node + List<String> children = w.getFieldAsList(TreeNode.EXT_TREENODE, + TreeNode.FIELD_CHILDREN, String.class); + List<String> newChildren = new ArrayList<String>(); + if(children != null) { + newChildren.addAll(children); + } + + // Find old document to do diff + SolrDocument treeNodeDoc = findById(treeNodeId); + if(treeNodeDoc != null) { + Collection oldChildren = treeNodeDoc. + getFieldValues(TreeNode.FQ_FIELD_CHILDREN + "_t"); + + // Remove old children not in current node + List<String> removeChildren = new ArrayList<String>(); + if(oldChildren != null) { + removeChildren.addAll(oldChildren); + removeChildren.removeAll(children); + for (String removeChild : removeChildren) { + solrServer.deleteByQuery(TREENODE_CHILD + ":" + removeChild); + } + + // Remove old child in new children + newChildren.removeAll(oldChildren); + } + + // Update path other node refer to current node + // FIXME jru 20091926 test change path + SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + treeNodeId + ":*"); + QueryResponse response = solrServer.query(query); + SolrDocumentList updateDocs = response.getResults(); + for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { + SolrDocument solrDocument = iterator.next(); + String child = (String) solrDocument.getFieldValue(TreeNode.EXT_TREENODE + ".wikittyId"); + // Child is not deleted + if(!removeChildren.contains(child)) { + SolrInputDocument doc = createTreeNodeDocument(treeNodeId, child, parentMap); + + // Copy other field after node + String parent = TREENODE_PREFIX + treeNodeId; + String childParent = (String) solrDocument.getFieldValue(parent); + while (childParent != null) { + doc.setField(parent, childParent); + parent = TREENODE_PREFIX + childParent; + childParent = (String) solrDocument.getFieldValue(parent); + } + result.add(doc); + } + } + } + + // New children + for (String newChild : newChildren) { + SolrInputDocument doc = createTreeNodeDocument(treeNodeId, newChild, parentMap); + result.add(doc); + } + + } + return result; + } + + /** + * Create a doc between node and child + */ + public SolrInputDocument createTreeNodeDocument(String treeNodeId, String child, ParentMap parentMap) { + SolrInputDocument doc = new SolrInputDocument(); + doc.setField(SOLR_ID, TreeNode.EXT_TREENODE + "-" + treeNodeId + "-" + child); + doc.setField(TREENODE_CHILD, child); + doc.setField(TREENODE_PREFIX + treeNodeId, TREENODE_EMPTY); + + // Add path + String childParent = treeNodeId; + String parent = parentMap.get(childParent); + while (parent != null) { + doc.setField(TREENODE_PREFIX + parent, childParent); + childParent = parent; + parent = parentMap.get(childParent); + } + doc.setField(TREENODE_ROOT, childParent); + + return doc; + } + + /** + * Find solr document by id + */ + protected SolrDocument findById(String id) { + SolrQuery query = new SolrQuery("id:" + id); + QueryResponse response; + try { + response = solrServer.query(query); + } catch (SolrServerException eee) { + throw new WikittyException(eee); + } + + SolrDocumentList results = response.getResults(); + long numFound = results.getNumFound(); + if(numFound == 1) { + return results.get(0); + } + + return null; + } + + static public String getSolrFieldName(String fqfieldName, FieldType fieldType) { + TYPE type = fieldType.getType(); + String result = getSolrFieldName(fqfieldName, type); + return result; + } + + public static String getSolrFieldName(String fqfieldName, TYPE type) { + switch (type) { + case BOOLEAN: + return fqfieldName + "_b"; + case DATE: + return fqfieldName + "_dt"; + case STRING: + return fqfieldName + "_t"; + case NUMERIC: + return fqfieldName + "_d"; + default: + return fqfieldName; + } + } +} Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/schema.xml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,308 @@ +<?xml version="1.0" encoding="UTF-8" ?> + + + + +<schema name="wikitty" version="1.1"> +  + + <types> +  + +  + <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/> + +  + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/> + +  + + +  + <fieldType name="integer" class="solr.IntField" omitNorms="true"/> + <fieldType name="long" class="solr.LongField" omitNorms="true"/> + <fieldType name="float" class="solr.FloatField" omitNorms="true"/> + <fieldType name="double" class="solr.DoubleField" omitNorms="true"/> + + +  + <fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/> + + +  + <fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/> + + +  + <fieldType name="random" class="solr.RandomSortField" indexed="true" /> + +  + +  + +  + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + </analyzer> + </fieldType> + +  + <fieldType name="text" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> +  +  + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="stopwords.txt" + enablePositionIncrements="true" + /> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + + <fieldType name="text_core" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory" /> + <filter class="solr.LowerCaseFilterFactory" /> +  + </analyzer> + </fieldType> + + +  + <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100" > + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + +  + <fieldType name="textSpell" class="solr.TextField" positionIncrementGap="100" > + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + +  + <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true"> + <analyzer> +  + <tokenizer class="solr.KeywordTokenizerFactory"/> +  + <filter class="solr.LowerCaseFilterFactory" /> +  + <filter class="solr.TrimFilterFactory" /> +  + <filter class="solr.PatternReplaceFilterFactory" + pattern="([^a-z])" replacement="" replace="all" + /> + </analyzer> + </fieldType> + +  + <fieldtype name="ignored" stored="false" indexed="false" class="solr.StrField" /> + + </types> + + + <fields> + <field name="id" type="string" indexed="true" stored="true" required="true" /> + <field name="extensions" type="string" indexed="true" stored="false" multiValued="true"/> + +  + <field name="text" type="text" indexed="true" stored="false" multiValued="true"/> + + <dynamicField name="Label.label*" type="text" indexed="true" stored="true" multiValued="true"/> + + <dynamicField name="*_i" type="sint" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_s" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_l" type="slong" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_t" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_b" type="boolean" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_f" type="sfloat" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_d" type="sdouble" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_dt" type="date" indexed="true" stored="true" multiValued="true"/> + +  + <dynamicField name="*" type="string" indexed="true" stored="true" multiValued="true"/> + +  + <dynamicField name="_*" type="string" indexed="true" stored="true" multiValued="true"/> + + </fields> + +  + <uniqueKey>id</uniqueKey> + +  + <defaultSearchField>id</defaultSearchField> + +  + <solrQueryParser defaultOperator="OR"/> + +</schema> Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/solrconfig.xml 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,700 @@ +<?xml version="1.0" encoding="UTF-8" ?> + + +<config> +  + <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError> + +  + <dataDir>${solr.data.dir:./solr/data}</dataDir> + + + <indexDefaults> +  + <useCompoundFile>false</useCompoundFile> + + <mergeFactor>10</mergeFactor> +  +  +  + <ramBufferSizeMB>32</ramBufferSizeMB> + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + <writeLockTimeout>1000</writeLockTimeout> + <commitLockTimeout>10000</commitLockTimeout> + +  +  +  +  + +  +  + +  + <lockType>single</lockType> + </indexDefaults> + + <mainIndex> +  + <useCompoundFile>false</useCompoundFile> + <ramBufferSizeMB>32</ramBufferSizeMB> + <mergeFactor>10</mergeFactor> +  +  + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + +  + <unlockOnStartup>false</unlockOnStartup> + </mainIndex> + +  + <jmx /> + +  + <updateHandler class="solr.DirectUpdateHandler2"> + +  + +  + +  +  +  + + </updateHandler> + + + <query> +  + <maxBooleanClauses>1024</maxBooleanClauses> + + +  + <filterCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="128"/> + +  + <queryResultCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="32"/> + +  + <documentCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + +  + <enableLazyFieldLoading>true</enableLazyFieldLoading> + +  +  + +  + +  + <queryResultWindowSize>50</queryResultWindowSize> + +  + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + +  + <HashDocSet maxSize="3000" loadFactor="0.75"/> + +  +  + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + +  + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static firstSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + +  + <useColdSearcher>false</useColdSearcher> + +  + <maxWarmingSearchers>2</maxWarmingSearchers> + + </query> + +  + <requestDispatcher handleSelect="true" > +  + <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" /> + +  +  + <httpCaching lastModifiedFrom="openTime" + etagSeed="Solr"> +  +  +  + </httpCaching> + </requestDispatcher> + + +  + <requestHandler name="standard" class="solr.SearchHandler" default="true"> +  + <lst name="defaults"> + <str name="echoParams">explicit</str> +  + </lst> + </requestHandler> + + +  + <requestHandler name="dismax" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <float name="tie">0.01</float> + <str name="qf"> + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + </str> + <str name="pf"> + text^0.2 features^1.1 name^1.5 manu^1.4 manu_exact^1.9 + </str> + <str name="bf"> + ord(popularity)^0.5 recip(rord(price),1,1000,1000)^0.3 + </str> + <str name="fl"> + id,name,price,score + </str> + <str name="mm"> + 2<-1 5<-2 6<90% + </str> + <int name="ps">100</int> + <str name="q.alt">*:*</str> +  + <str name="hl.fl">text features name</str> +  + <str name="f.name.hl.fragsize">0</str> +  + <str name="f.name.hl.alternateField">name</str> + <str name="f.text.hl.fragmenter">regex</str>  + </lst> + </requestHandler> + +  + <requestHandler name="partitioned" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <str name="qf">text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0</str> + <str name="mm">2<-1 5<-2 6<90%</str> +  + <str name="bq">incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2</str> + </lst> +  + <lst name="appends"> + <str name="fq">inStock:true</str> + </lst> +  + <lst name="invariants"> + <str name="facet.field">cat</str> + <str name="facet.field">manu_exact</str> + <str name="facet.query">price:[* TO 500]</str> + <str name="facet.query">price:[500 TO *]</str> + </lst> + </requestHandler> + + +  + +  + <searchComponent name="spellcheck" class="solr.SpellCheckComponent"> + + <str name="queryAnalyzerFieldType">textSpell</str> + + <lst name="spellchecker"> + <str name="name">default</str> + <str name="field">spell</str> + <str name="spellcheckIndexDir">./spellchecker1</str> + + </lst> + <lst name="spellchecker"> + <str name="name">jarowinkler</str> + <str name="field">spell</str> +  + <str name="distanceMeasure">org.apache.lucene.search.spell.JaroWinklerDistance</str> + <str name="spellcheckIndexDir">./spellchecker2</str> + + </lst> + + <lst name="spellchecker"> + <str name="classname">solr.FileBasedSpellChecker</str> + <str name="name">file</str> + <str name="sourceLocation">spellings.txt</str> + <str name="characterEncoding">UTF-8</str> + <str name="spellcheckIndexDir">./spellcheckerFile</str> + </lst> + </searchComponent> + +  + <requestHandler name="/spellCheckCompRH" class="solr.SearchHandler"> + <lst name="defaults"> +  + <str name="spellcheck.onlyMorePopular">false</str> +  + <str name="spellcheck.extendedResults">false</str> +  + <str name="spellcheck.count">1</str> + </lst> + <arr name="last-components"> + <str>spellcheck</str> + </arr> + </requestHandler> + +  + + +  + + +  + <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" /> + +  + <requestHandler name="/analysis" class="solr.AnalysisRequestHandler" /> + + +  + <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" /> + + +  + <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" /> + +  + <requestHandler name="/admin/ping" class="PingRequestHandler"> + <lst name="defaults"> + <str name="qt">standard</str> + <str name="q">solrpingquery</str> + <str name="echoParams">all</str> + </lst> + </requestHandler> + +  + <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" > + <lst name="defaults"> + <str name="echoParams">explicit</str>  + <str name="echoHandler">true</str> + </lst> + </requestHandler> + + <highlighting> +  +  + <fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true"> + <lst name="defaults"> + <int name="hl.fragsize">100</int> + </lst> + </fragmenter> + +  + <fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter"> + <lst name="defaults"> +  + <int name="hl.fragsize">70</int> +  + <float name="hl.regex.slop">0.5</float> +  + <str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str> + </lst> + </fragmenter> + +  + <formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true"> + <lst name="defaults"> + <str name="hl.simple.pre"><![CDATA[]]></str> + <str name="hl.simple.post"><![CDATA[]]></str> + </lst> + </formatter> + </highlighting> + + +  + +  + <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter"> + <int name="xsltCacheLifetimeSeconds">5</int> + </queryResponseWriter> + + + <queryParser name="wikitty" class="org.sharengo.wikitty.solr.WikittyQueryParser"/> + +  + +  + +  + <admin> + <defaultQuery>solr</defaultQuery> + +  + </admin> + +</config> Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/main/resources/stopwords.txt 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + Added: wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties (rev 0) +++ wikengo_core-wikitty/wikengo_core-wikitty-solr-impl/src/test/resources/log4j.properties 2009-11-04 10:20:45 UTC (rev 311) @@ -0,0 +1,9 @@ +# Appender and Layout +log4j.appender.logConsole=org.apache.log4j.ConsoleAppender +log4j.appender.logConsole.layout=org.apache.log4j.PatternLayout +log4j.appender.logConsole.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Configuration by components +log4j.rootLogger=ERROR, logConsole +log4j.category.org.sharengo.wikitty=DEBUG +#log4j.category.org.apache.solr=DEBUG

1 0

r310 - wikengo_core-wikitty
by kmorin＠users.nuiton.org 04 Nov '09

04 Nov '09

Author: kmorin Date: 2009-11-04 11:20:31 +0100 (Wed, 04 Nov 2009) New Revision: 310 Modified: wikengo_core-wikitty/pom.xml Log: Separation of solr Modified: wikengo_core-wikitty/pom.xml =================================================================== --- wikengo_core-wikitty/pom.xml 2009-11-04 10:20:09 UTC (rev 309) +++ wikengo_core-wikitty/pom.xml 2009-11-04 10:20:31 UTC (rev 310) @@ -14,6 +14,7 @@ <modules> <module>wikengo_core-wikitty-api</module> <module>wikengo_core-wikitty-jdbc-impl</module> + <module>wikengo_core-wikitty-solr-impl</module> </modules> <parent> @@ -30,9 +31,9 @@ <snapshotRepository> <id>nuiton</id> <url>scpexe://nuiton.org/var/lib/maven/snapshot</url> - </snapshotRepository> + </snapshotRepository> </distributionManagement> - +    @@ -77,7 +78,7 @@ </repository> </repositories> - + <profiles>  <profile>

1 0

r309 - wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl
by kmorin＠users.nuiton.org 04 Nov '09

04 Nov '09

Author: kmorin Date: 2009-11-04 11:20:09 +0100 (Wed, 04 Nov 2009) New Revision: 309 Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/pom.xml Log: Separation of solr Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/pom.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/pom.xml 2009-11-04 10:19:47 UTC (rev 308) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/pom.xml 2009-11-04 10:20:09 UTC (rev 309) @@ -27,6 +27,11 @@ </dependency> <dependency> <groupId>${project.groupId}</groupId> + <artifactId>core-wikitty-solr-impl</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> <artifactId>core-wikitty-test</artifactId> <version>1.0-SNAPSHOT</version> </dependency>

1 0

r308 - in wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src: main/java/org/sharengo/wikitty/jdbc main/resources test/java/org/sharengo/wikitty/jdbc test/resources
by kmorin＠users.nuiton.org 04 Nov '09

04 Nov '09

Author: kmorin Date: 2009-11-04 11:19:47 +0100 (Wed, 04 Nov 2009) New Revision: 308 Added: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEngineTest.java Removed: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/WikittySearchEnginTest.java Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/JDBCStorageTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/java/org/sharengo/wikitty/jdbc/TreeTest.java wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/jdbc.properties wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/test/resources/solrconfig.xml Log: Separation of solr Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/Restriction2Solr.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,389 +0,0 @@ -package org.sharengo.wikitty.jdbc; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.mortbay.log.Log; -import org.sharengo.wikitty.WikittyException; -import org.sharengo.wikitty.search.*; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author "Nicolas Chapurlat" <nicolas.chapurlat(a)logica.com> - * @author "Guillaume Dufrêne" <dufrene(a)argia.fr> - * - * This class is used to parse Restriction to create lucene request on - * content. Every operators describe in RestrictionName is handle. Parsing may - * throw exception when restriction parameters are incorrect. - */ -public class Restriction2Solr { - - private static final int MAX_SUBQUERY_RESULT = 100; - - /** - * use to permit client to modify fieldname during query generation - */ - static public interface FieldModifier { - public String convertToSolr(String fieldname); - - public String convertToField(String solrName); - } - - final static protected FieldModifier dummyFieldModifier = new FieldModifier() { - public String convertToSolr(String fieldname) { - return fieldname; - } - - public String convertToField(String solrName) { - return solrName; - } - }; - - protected FieldModifier fieldModifer; - - public Restriction2Solr() { - this(dummyFieldModifier); - } - - public Restriction2Solr(FieldModifier fieldModifer) { - this.fieldModifer = fieldModifer; - } - - public String toSolr(Restriction restriction) { - return toSolr(restriction, null); - } - - public String toSolr(Restriction restriction, SolrServer solr) - throws WikittyException { - // ParameterValidator.checkNullParameter(restriction, "restriction"); - switch (restriction.getName()) { - case TRUE: - return true2solr(); - case FALSE: - return false2solr(); - case NOT: - Not not = (Not) restriction; - return not2solr(not); - case AND: - And and = (And) restriction; - return and2solr(and); - case OR: - Or or = (Or) restriction; - return or2solr(or); - case EQUALS: - Equals eq = (Equals) restriction; - return eq2solr(eq); - case NOT_EQUALS: - NotEquals neq = (NotEquals) restriction; - return neq2solr(neq); - case LESS: - Less less = (Less) restriction; - return less2solr(less); - case LESS_OR_EQUAL: - LessOrEqual lessEq = (LessOrEqual) restriction; - return lessEq2solr(lessEq); - case GREATER: - Greater great = (Greater) restriction; - return great2solr(great); - case GREATER_OR_EQUAL: - GreaterOrEqual greatEq = (GreaterOrEqual) restriction; - return greatEq2solr(greatEq); - case BETWEEN: - Between between = (Between) restriction; - return between2solr(between); - case CONTAINS: - Contains contains = (Contains) restriction; - return contains2solr(contains); - case IN: - In in = (In) restriction; - return in2solr(in); - case STARTS_WITH: - StartsWith start = (StartsWith) restriction; - return start2solr(start); - case ENDS_WITH: - EndsWith end = (EndsWith) restriction; - return end2solr(end); - case ASSOCIATED: - AssociatedRestriction associated = (AssociatedRestriction) restriction; - return associated2solr(associated, solr); - case KEYWORD: - Keyword keyword = (Keyword) restriction; - return keyword2solr(keyword); - default: - throw new WikittyException("this kind of restriction is not supported : " - + restriction.getName().toString()); - } - } - - private String in2solr(In in) { - boolean first = true; - String result = in.getElement().getName() + ":["; - for (String value : in.getValue()) { - if (!first) { - result += ", "; - first = false; - } - result += value; - } - result += "]"; - return result; - } - - private String associated2solr(AssociatedRestriction associated, SolrServer solr) throws WikittyException { - String subQuery = toSolr(associated.getRestriction()); - SolrQuery query = new SolrQuery(subQuery); - query.setRows(MAX_SUBQUERY_RESULT); - QueryResponse resp = null; - try { - Log.info("Execute sub-query : " + subQuery); - resp = solr.query(query); - } catch (SolrServerException e) { - throw new WikittyException("Unable to execute associative query on " + associated.getElement().getName(), e); - } - SolrDocumentList solrResults = resp.getResults(); - - Restriction generatedRestriction = null; - long size = solrResults.size(); - if (size == 0) { - throw new WikittyException("Associated " + associated.getElement().getName() + " do not retrieved any result"); - } - if (size == 1) { - generatedRestriction = RestrictionHelper.eq(associated.getElement(), (String) solrResults.get(0).getFieldValue("id")); - } else { - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue("id"); // FIXME : set a constant field reference - ids.add(id); - } - generatedRestriction = new In(associated.getElement(), ids); - } - Restriction parent = associated.getParentRestrictionDto(); - And and; - if (parent instanceof And) { - and = (And) parent; - and.getRestrictions().add(generatedRestriction); - } else { - and = RestrictionHelper.and(Arrays.asList(new Restriction[]{associated.getParentRestrictionDto(), generatedRestriction})); - } - return toSolr(and); - } - - private String not2solr(Not not) throws WikittyException { - if (not.getRestriction() == null) { - throw new WikittyException("not.restriction"); - } - return "( *:* - " + toSolr(not.getRestriction()) + " )"; - } - - private String and2solr(And and) throws WikittyException { - if (and.getRestrictions() == null) { - throw new WikittyException("and.restrictions is null"); - } - if (and.getRestrictions().size() < 2) { - throw new WikittyException("AND is an operator that handle 2 operand at least"); - } - boolean first = true; - StringBuffer result = new StringBuffer(); - for (Restriction restriction : and.getRestrictions()) { - if (first) { - result.append("( ").append(toSolr(restriction)); - first = false; - } else { - result.append(" AND ").append(toSolr(restriction)); - } - } - return result.append(" )").toString(); - } - - private String or2solr(Or or) throws WikittyException { - if (or.getRestrictions() == null) { - throw new WikittyException("or.restrictions is null"); - } - if (or.getRestrictions().size() < 2) { - throw new WikittyException("OR is an operator that handle 2 operand at least"); - } - boolean first = true; - StringBuffer result = new StringBuffer(); - for (Restriction restriction : or.getRestrictions()) { - if (first) { - result.append("( "); - first = false; - } else { - result.append(" OR "); - } - result.append(toSolr(restriction)); - } - return result.append(" )").toString(); - } - - private String eq2solr(Equals eq) throws WikittyException { - return element2solr(eq.getElement()) + ":" + value2solr(eq.getValue()); - } - - private String neq2solr(NotEquals neq) - throws WikittyException { - return "( *:* - " + element2solr(neq.getElement()) + ":" - + value2solr(neq.getValue()) + " )"; - } - - private String less2solr(Less less) throws WikittyException { - return element2solr(less.getElement()) + ":{* TO " - + value2solr(less.getValue()) + "}"; - } - - private String lessEq2solr(LessOrEqual lessEq) - throws WikittyException { - return element2solr(lessEq.getElement()) + ":[* TO " - + value2solr(lessEq.getValue()) + "]"; - } - - private String great2solr(Greater great) - throws WikittyException { - return element2solr(great.getElement()) + ":{" - + value2solr(great.getValue()) + " TO *}"; - } - - private String greatEq2solr(GreaterOrEqual greatEq) - throws WikittyException { - return element2solr(greatEq.getElement()) + ":[" - + value2solr(greatEq.getValue()) + " TO *]"; - } - - private String between2solr(Between between) - throws WikittyException { - if (between.getElement() == null) { - throw new WikittyException("contains.element"); - } - if (between.getMin() == null) { - throw new WikittyException("contains.min"); - } - if (between.getMax() == null) { - throw new WikittyException("contains.max"); - } - return element2solr(between.getElement()) + ":[" - + value2solr(between.getMin()) + " TO " - + value2solr(between.getMax()) + "]"; - } - - private String contains2solr(Contains contains) - throws WikittyException { - if (contains.getElement() == null) { - throw new WikittyException("contains.element"); - } - if (contains.getValue() == null) { - throw new WikittyException("contains.values"); - } - if (contains.getValue().size() < 1) { - throw new WikittyException("CONTAINS is an operator that handle 1 operand at least"); - } - boolean first = true; - StringBuffer result = new StringBuffer(); - for (String value : contains.getValue()) { - if (first) { - result.append("( "); - first = false; - } else { - result.append(" AND "); - } - result.append("( "); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); - result.append(" OR "); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); - result.append(" OR "); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); - result.append(" )"); - } - return result.append(" )").toString(); - } - - private String start2solr(StartsWith start) - throws WikittyException { - return element2solr(start.getElement()) + ":" - + value2solr(start.getValue(), true, false); - } - - private String end2solr(EndsWith end) { - return element2solr(end.getElement()) + ":" + value2solr(end.getValue()); - } - - private String true2solr() { - return "( *:* )"; - } - - private String false2solr() { - return "( *:* - *:* )"; - } - - private String keyword2solr(Keyword keyword) { - return value2solr(keyword.getValue()); - } - - private String value2solr(String value) { - return value2solr(value, false, false); - } - - private String value2solr(String value, boolean starOnEnd, - boolean revert) { - String result; - if (value != null) { - if (value.equals("*")) { - return "*"; - } - - if (revert) { - result = Restriction2Solr.escapeValue(reverse(value)); - } else { - result = Restriction2Solr.escapeValue(value); - // result = value; - } - } else { - result = ""; - } - if (starOnEnd) { - result += "*"; - } - if (result.contains(" ")) { - result = "\"" + result + "\""; - } - return result; - } - - private String element2solr(Element element) throws WikittyException { - String result = element.getName(); - if (Element.ELT_EXTENSION.equals(result)) { - result = WikittySearchEnginSolr.SOLR_EXTENSIONS; - } else if (Element.ELT_ID.equals(result)) { - result = WikittySearchEnginSolr.SOLR_ID; - } else { - result = element.getName(); - } - result = fieldModifer.convertToSolr(result); - return result; - } - - private static String escapeValue(String value) { - final String LUCENE_REPLACE_PATTERN = "\\+" + "|-" + "|&&" + "|\\|\\|" - + "|!" + "|\$|\$" + "|\\[|\\]" + "|\\{|\\}" + "|\\^" + "|\"" - + "|\\~" + "|\\*" + "|\\?" + "|:" + "|\\\\"; - return value.replaceAll(LUCENE_REPLACE_PATTERN, "\\\\$0"); - } - - private String reverse(String source) { - int i, len = source.length(); - StringBuffer dest = new StringBuffer(len); - - for (i = (len - 1); i >= 0; i--) - dest.append(source.charAt(i)); - return dest.toString(); - } -} - Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyQueryParser.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,71 +0,0 @@ -package org.sharengo.wikitty.jdbc; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.Query; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.LuceneQParserPlugin; -import org.apache.solr.search.QParser; -import org.apache.solr.search.QueryParsing; -import org.apache.solr.search.SolrQueryParser; - - -/** - * Add allow leading wildcard - * setAllowLeadingWildcard(true); - * setLowercaseExpandedTerms(true); - * Example: <code>{!wikitty q.op=AND df=text sort='price asc'}myfield:foo +bar -baz</code> - * More information @see LuceneQParserPlugin - */ -public class WikittyQueryParser extends LuceneQParserPlugin { - - public static String NAME = "wikitty"; - - @Override - public void init(NamedList args) { - } - - @Override - public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { - return new SolrQParser(qstr, localParams, params, req); - } -} - -class SolrQParser extends QParser { - - static private Log log = LogFactory.getLog(SolrQParser.class); - String sortStr; - SolrQueryParser lparser; - - public SolrQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { - super(qstr, localParams, params, req); - } - - @Override - public Query parse() throws ParseException { - - String defaultField = getParam(CommonParams.DF); - lparser = new SolrQueryParser(this, defaultField); - lparser.setAllowLeadingWildcard(true); - lparser.setLowercaseExpandedTerms(true); - - String opParam = getParam(QueryParsing.OP); - if (opParam != null) { - lparser.setDefaultOperator("AND".equals(opParam) ? QueryParser.Operator.AND : QueryParser.Operator.OR); - } - - String qstr = getString(); - log.debug("Query parse : " + qstr); - return lparser.parse(qstr); - } - - @Override - public String[] getDefaultHighlightFields() { - return new String[]{lparser.getField()}; - } -} Deleted: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittySearchEnginSolr.java 2009-11-04 10:19:47 UTC (rev 308) @@ -1,659 +0,0 @@ -/* *##% - * Copyright (c) 2009 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.sharengo.wikitty.jdbc; - - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.core.CoreContainer; -import org.sharengo.wikitty.*; -import org.sharengo.wikitty.FieldType.TYPE; -import org.sharengo.wikitty.search.Search; - -import java.io.IOException; -import java.util.*; - -/** - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class WikittySearchEnginSolr implements WikittySearchEngin { - - /** - * to use log facility, just put in your code: log.info(\"...\"); - */ - static private Log log = LogFactory.getLog(WikittySearchEnginSolr.class); - - /** - * id field in solr - */ - static final protected String SOLR_ID = "id"; - - /** - * extensions field name in solr - */ - static final public String SOLR_EXTENSIONS = "extensions"; - - /** - * extension use to store field without extension to search on all extesnion - */ - static final public String SOLR_ALL_EXTENSIONS = "all"; - - /** - * Precise the query parser to use, is allow leading wildcard - */ - static final public String SOLR_QUERY_PARSER = "{!wikitty}"; - - // Use for indexation tree node - static final public String TREENODE_PREFIX = TreeNode.EXT_TREENODE + "."; - static final public String TREENODE_CHILD = TREENODE_PREFIX + "wikittyId"; - static final public String TREENODE_EMPTY = TREENODE_PREFIX + "empty"; - static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; - - static protected class CommandSolr implements Command { - /** - * if is store command toStore is true, if is delete toStore is false - */ - boolean toStore; - String id; - Wikitty wikitty; - - /** - * use to store command - */ - public CommandSolr(Wikitty wikitty) { - this.wikitty = wikitty; - this.toStore = true; - } - - /** - * use for deletion command - */ - public CommandSolr(String id) { - this.id = id; - this.toStore = false; - } - } - - static protected class TypeFieldModifer implements Restriction2Solr.FieldModifier { - protected WikittyExtensionStorage extensionStorage; - - public TypeFieldModifer(WikittyExtensionStorage extensionStorage) { - this.extensionStorage = extensionStorage; - } - - public String convertToSolr(String fqfieldname) { - String result = fqfieldname; - String[] searchField = fqfieldname.split("\\."); - - if (searchField.length == 2) { - String extName = searchField[0]; - String fieldName = searchField[1]; - - String version = extensionStorage.getLastVersion(extName); - if (version != null) { // not valid extension is version == null - WikittyExtension ext = extensionStorage.restore( - WikittyExtension.computeId(extName, version)); - FieldType type = ext.getFieldType(fieldName); - if (type != null) { // type can be null if extension version differ - result = WikittySearchEnginSolr.getSolrFieldName( - fqfieldname, type); - } else { - log.debug(String.format( - "Field '%s' could not be found in '%s'", - fqfieldname, extName)); - } - } else { - log.warn("No version for extension " + extName); - } - } - - if (searchField.length == 3) { - String extName = searchField[0]; - String fieldName = searchField[1]; - String fieldType = searchField[2]; - - if (Criteria.ALL_EXTENSIONS.equals(extName)) { - TYPE type = FieldType.TYPE.valueOf(fieldType); - String solrFieldName = SOLR_ALL_EXTENSIONS + "." + fieldName; - result = WikittySearchEnginSolr.getSolrFieldName(solrFieldName, type); - } else { - log.warn("All extension not found " + extName); - } - } - - return result; - } - - public String convertToField(String solrName) { - String fieldname = solrName.replaceAll("(_b$)|(_dt$)|(_t$)|(_l$)", ""); - return fieldname; - } - - } - - /** - * Store parent for each node and search in index if not known parent - */ - protected static class ParentMap extends HashMap<String, String> { - - protected WikittySearchEnginSolr searchEnginSolr; - - public ParentMap(WikittySearchEnginSolr searchEnginSolr) { - this.searchEnginSolr = searchEnginSolr; - } - - @Override - public String get(Object key) { - String id = String.valueOf(key); - String parent = super.get(id); - - // If not found in map, search in index - if (parent == null) { - SolrDocument doc = searchEnginSolr.findById(id); - if (doc == null) { - // is root - return null; - } - parent = (String) doc.getFieldValue(TreeNode.FQ_FIELD_PARENT); - put(id, parent); - } - return parent; - } - - /** - * Add parent map in transaction if is not exists and return it - */ - public static ParentMap getParentMap( - WikittySearchEnginSolr searchEnginSolr, - WikittyTransaction transaction) { - ParentMap value = (ParentMap) transaction.getTagValue(ParentMap.class); - if (value == null) { - value = new ParentMap(searchEnginSolr); - transaction.setTagValue(ParentMap.class, value); - } - return value; - } - } - - /** - * solr server - */ - protected SolrServer solrServer; - /** - * Criteria transformer to solr - */ - protected Restriction2Solr restriction2Solr; - /** - * Field modifier use to transform to solr format - */ - protected TypeFieldModifer fieldModifier; - - public WikittySearchEnginSolr(WikittyExtensionStorage extensionStorage) { - super(); - try { - CoreContainer.Initializer initializer = new CoreContainer.Initializer(); - CoreContainer coreContainer = initializer.initialize(); - solrServer = new EmbeddedSolrServer(coreContainer, ""); - - fieldModifier = new TypeFieldModifer(extensionStorage); - restriction2Solr = new Restriction2Solr(fieldModifier); - - } catch (Exception eee) { - throw new WikittyException(eee); - } - } - - public void clear() { - try { - solrServer.deleteByQuery("*:*"); - solrServer.commit(true, true); - } catch (Exception eee) { - throw new WikittyException(eee); - } - } - - public List<Command> prepare(WikittyTransaction transaction, - Collection<Wikitty> wikitties) { - List<Command> result = new ArrayList<Command>(wikitties.size()); - for (Wikitty w : wikitties) { - - // Store known parent in transaction - if (w.hasExtension(TreeNode.EXT_TREENODE)) { - ParentMap parentMap = ParentMap.getParentMap(this, transaction); - String wikittyId = w.getId(); - String parentId = w.getFieldAsWikitty( - TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); - parentMap.put(wikittyId, parentId); - } - - result.add(new CommandSolr(w)); - } - return result; - } - - public UpdateResponse commit(WikittyTransaction transaction, - List<Command> wikittyIndexationCommandList) { - try { - for (Command c : wikittyIndexationCommandList) { - CommandSolr cs = (CommandSolr) c; - if (cs.toStore) { - SolrInputDocument doc = createIndexDocument(cs.wikitty); - solrServer.add(doc); - - Collection<SolrInputDocument> docs = createTreeNodeIndex(transaction, cs.wikitty); - if (!docs.isEmpty()) { - solrServer.add(docs); - } - - } else { - // Delete all reference on tree - solrServer.deleteByQuery(TREENODE_CHILD + ":" + cs.id); - // FIXME: jru 20091027 delete node in middle in tree - solrServer.deleteByQuery(TREENODE_PREFIX + cs.id + ":" + TREENODE_EMPTY); - - solrServer.deleteById(cs.id); - } - } - solrServer.commit(); - - // no specific result needed - UpdateResponse result = new UpdateResponse(); - return result; - } catch (Exception eee) { - throw new WikittyException(eee); - } - } - - public List<Command> delete(WikittyTransaction transaction, - List<String> idList) throws WikittyException { - List<Command> result = new ArrayList<Command>(idList.size()); - for (String id : idList) { - result.add(new CommandSolr(id)); - } - return result; - } - - public PagedResult<String> findAllByCriteria(Criteria criteria) { - try { - - // Create query with restriction - String queryString = restriction2Solr.toSolr(criteria.getRestriction(), solrServer); - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + queryString); - - // Add pagged - int firstIndex = criteria.getFirstIndex(); - int endIndex = criteria.getEndIndex(); - - query.setStart(firstIndex); - int nbRows; - if (endIndex == -1) { - // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR - nbRows = Integer.MAX_VALUE - firstIndex; - } else { - nbRows = endIndex - firstIndex + 1; - } - query.setRows(nbRows); - - // Add sorting - List<String> sortAscending = criteria.getSortAscending(); - if (sortAscending != null) { - for (String sort : sortAscending) { - String tranform = fieldModifier.convertToSolr(sort); - query.setSortField(tranform, SolrQuery.ORDER.asc); - } - } - - List<String> sortDescending = criteria.getSortDescending(); - if (sortDescending != null) { - for (String sort : sortDescending) { - String tranform = fieldModifier.convertToSolr(sort); - query.setSortField(tranform, SolrQuery.ORDER.desc); - } - } - - // Add faceting - List<String> facetField = criteria.getFacetField(); - List<Criteria> facetCriteria = criteria.getFacetCriteria(); - - // use to map query string to criteria facet name - Map<String, String> facetQueryToName = new HashMap<String, String>(); - - if ((facetField != null && !facetField.isEmpty()) - || (facetCriteria != null && !facetCriteria.isEmpty())) { - query.setFacet(true); - query.setFacetMinCount(1); - // query.setFacetLimit(8); // no limit actualy - - // field facetisation - if (facetField != null) { - for (String fqfieldName : facetField) { - String tranform = fieldModifier.convertToSolr(fqfieldName); - query.addFacetField(tranform); - } - } - - // query facetisation - if (facetCriteria != null) { - for (Criteria facet : facetCriteria) { - String queryFacet = - restriction2Solr.toSolr(facet.getRestriction()); - facetQueryToName.put(queryFacet, facet.getName()); - query.addFacetQuery(queryFacet); - } - } - } - - QueryResponse resp = solrServer.query(query); - SolrDocumentList solrResults = resp.getResults(); - - Map<String, List<FacetTopic>> facets = new HashMap<String, List<FacetTopic>>(); - if (facetField != null && !facetField.isEmpty()) { - for (FacetField facet : resp.getFacetFields()) { - String facetName = fieldModifier.convertToField(facet.getName()); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - if (facet.getValues() != null) { - for (FacetField.Count value : facet.getValues()) { - String topicName = value.getName(); - if (!topicName.endsWith(TREENODE_EMPTY)) { - int topicCount = (int) value.getCount(); - FacetTopic topic = new FacetTopic(facetName, topicName, topicCount); - topics.add(topic); - } - } - } - facets.put(facetName, topics); - } - } - if (facetCriteria != null && !facetCriteria.isEmpty()) { - for (Map.Entry<String, Integer> facet : resp.getFacetQuery().entrySet()) { - String facetName = facet.getKey(); - // don't use contains because, map can have key with null value - if (null != facetQueryToName.get(facetName)) { - facetName = facetQueryToName.get(facetName); - } - Integer count = facet.getValue(); - List<FacetTopic> topics = new ArrayList<FacetTopic>(); - FacetTopic topic = new FacetTopic(facetName, facetName, count); - topics.add(topic); - facets.put(facetName, topics); - } - } - - List<String> ids = new ArrayList<String>(solrResults.size()); - for (SolrDocument doc : solrResults) { - String id = (String) doc.getFieldValue(SOLR_ID); - ids.add(id); - } - - int numFound = (int) resp.getResults().getNumFound(); - PagedResult<String> result = new PagedResult<String>( - firstIndex, numFound, queryString, facets, ids); - - return result; - } catch (SolrServerException e) { - throw new WikittyException(e); - } - } - - public Map<String, Integer> findAllChildren(Wikitty w) { - String wikittyId = w.getId(); - - String parent = w.getFieldAsWikitty(TreeNode.EXT_TREENODE, TreeNode.FIELD_PARENT); - if (parent == null) { - parent = TREENODE_ROOT; - } else { - parent = TREENODE_PREFIX + parent; - } - - // Find count with facet, if the node not contain recurcively content, - // the node not found with facet - Criteria criteria = Search.query().eq(parent, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(0) - .addFacetField(TREENODE_PREFIX + wikittyId); - PagedResult<String> search = findAllByCriteria(criteria); - - Map<String, Integer> counts = new HashMap<String, Integer>(); - List<FacetTopic> topics = search.getTopic(TREENODE_PREFIX + wikittyId); - if (topics != null) { - for (FacetTopic topic : topics) { - String topicName = topic.getTopicName(); - int topicCount = topic.getCount(); - counts.put(topicName, topicCount); - } - } - - log.debug("Facet result " + counts); - - // Find all children, add the other node not found with facet - criteria = Search.query().eq(TreeNode.FQ_FIELD_PARENT, wikittyId).criteria() - .setFirstIndex(0).setEndIndex(Criteria.ALL_ELEMENTS); - search = findAllByCriteria(criteria); - - List<String> children = search.getAll(); - for (String child : children) { - if (!counts.containsKey(child)) { - counts.put(child, 0); - } - } - - return counts; - } - - /** - * Create all index document to used to modify indexation. - * this method don't modify index - * - * @param wikitties all wikitties object to index - * @return list of SolrInputDocument used to modify index - */ - protected SolrInputDocument createIndexDocument(Wikitty w) { - log.debug("index wikitty " + w.getId()); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField(SOLR_ID, w.getId()); - - for (String name : w.getExtensionNames()) { - doc.addField(SOLR_EXTENSIONS, name); - } - - for (String fqfieldName : w.fieldNames()) { - FieldType type = w.getFieldType(fqfieldName); - // do not index wikitty field reference -// if (type.getType() == FieldType.TYPE.WIKITTY) { -// continue; -// } - String solrFqFieldName = getSolrFieldName(fqfieldName, type); - String[] extAndName = solrFqFieldName.split("\\."); - String solrFieldName = SOLR_ALL_EXTENSIONS + "." + extAndName[1]; - if (type.isCollection()) { - Collection collection = (Collection) w.getFqField(fqfieldName); - for (Object objectValue : collection) { - if (objectValue != null) { - // String value = WikittyUtil.toString(type, objectValue); - Object value = objectValue; - doc.addField(solrFqFieldName, value); - doc.addField(solrFieldName, value); - } - } - } else { - Object objectValue = w.getFqField(fqfieldName); - if (objectValue != null) { - // String value = WikittyUtil.toString(type, objectValue); - Object value = objectValue; - log.debug("index field " + solrFqFieldName + " with value '" + value + "'"); - doc.addField(solrFqFieldName, value); - doc.addField(solrFieldName, value); - } - } - } - - return doc; - } - - /** - * Create a doc for each child for a tree node, his contains the path for - * the child. Update old doc when the node is moved and child is deleted. - */ - protected Collection<SolrInputDocument> createTreeNodeIndex( - WikittyTransaction transaction, Wikitty w) - throws IOException, SolrServerException { - - Collection<SolrInputDocument> result = new ArrayList<SolrInputDocument>(); - if (w.hasExtension(TreeNode.EXT_TREENODE)) { - String treeNodeId = w.getId(); - ParentMap parentMap = (ParentMap) transaction.getTagValue(ParentMap.class); - - // Init new children with all children in node - List<String> children = w.getFieldAsList(TreeNode.EXT_TREENODE, - TreeNode.FIELD_CHILDREN, String.class); - List<String> newChildren = new ArrayList<String>(); - if (children != null) { - newChildren.addAll(children); - } - - // Find old document to do diff - SolrDocument treeNodeDoc = findById(treeNodeId); - if (treeNodeDoc != null) { - Collection oldChildren = treeNodeDoc. - getFieldValues(TreeNode.FQ_FIELD_CHILDREN + "_t"); - - // Remove old children not in current node - List<String> removeChildren = new ArrayList<String>(); - if (oldChildren != null) { - removeChildren.addAll(oldChildren); - removeChildren.removeAll(children); - for (String removeChild : removeChildren) { - solrServer.deleteByQuery(TREENODE_CHILD + ":" + removeChild); - } - - // Remove old child in new children - newChildren.removeAll(oldChildren); - } - - // Update path other node refer to current node - // FIXME jru 20091926 test change path - SolrQuery query = new SolrQuery(SOLR_QUERY_PARSER + TREENODE_PREFIX + treeNodeId + ":*"); - QueryResponse response = solrServer.query(query); - SolrDocumentList updateDocs = response.getResults(); - for (Iterator<SolrDocument> iterator = updateDocs.iterator(); iterator.hasNext();) { - SolrDocument solrDocument = iterator.next(); - String child = (String) solrDocument.getFieldValue(TreeNode.EXT_TREENODE + ".wikittyId"); - // Child is not deleted - if (!removeChildren.contains(child)) { - SolrInputDocument doc = createTreeNodeDocument(treeNodeId, child, parentMap); - - // Copy other field after node - String parent = TREENODE_PREFIX + treeNodeId; - String childParent = (String) solrDocument.getFieldValue(parent); - while (childParent != null) { - doc.setField(parent, childParent); - parent = TREENODE_PREFIX + childParent; - childParent = (String) solrDocument.getFieldValue(parent); - } - result.add(doc); - } - } - } - - // New children - for (String newChild : newChildren) { - SolrInputDocument doc = createTreeNodeDocument(treeNodeId, newChild, parentMap); - result.add(doc); - } - - } - return result; - } - - /** - * Create a doc between node and child - */ - public SolrInputDocument createTreeNodeDocument(String treeNodeId, String child, ParentMap parentMap) { - SolrInputDocument doc = new SolrInputDocument(); - doc.setField(SOLR_ID, TreeNode.EXT_TREENODE + "-" + treeNodeId + "-" + child); - doc.setField(TREENODE_CHILD, child); - doc.setField(TREENODE_PREFIX + treeNodeId, TREENODE_EMPTY); - - // Add path - String childParent = treeNodeId; - String parent = parentMap.get(childParent); - while (parent != null) { - doc.setField(TREENODE_PREFIX + parent, childParent); - childParent = parent; - parent = parentMap.get(childParent); - } - doc.setField(TREENODE_ROOT, childParent); - - return doc; - } - - /** - * Find solr document by id - */ - protected SolrDocument findById(String id) { - SolrQuery query = new SolrQuery("id:" + id); - QueryResponse response; - try { - response = solrServer.query(query); - } catch (SolrServerException eee) { - throw new WikittyException(eee); - } - - SolrDocumentList results = response.getResults(); - long numFound = results.getNumFound(); - if (numFound == 1) { - return results.get(0); - } - - return null; - } - - static public String getSolrFieldName(String fqfieldName, FieldType fieldType) { - TYPE type = fieldType.getType(); - String result = getSolrFieldName(fqfieldName, type); - return result; - } - - public static String getSolrFieldName(String fqfieldName, TYPE type) { - switch (type) { - case BOOLEAN: - return fqfieldName + "_b"; - case DATE: - return fqfieldName + "_dt"; - case STRING: - return fqfieldName + "_t"; - case NUMERIC: - return fqfieldName + "_d"; - default: - return fqfieldName; - } - } -} Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/java/org/sharengo/wikitty/jdbc/WikittyServiceJDBC.java 2009-11-04 10:19:47 UTC (rev 308) @@ -19,10 +19,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.sharengo.wikitty.AbstractWikittyService; -import org.sharengo.wikitty.WikittyExtensionStorage; -import org.sharengo.wikitty.WikittySearchEngin; -import org.sharengo.wikitty.WikittyStorage; +import org.sharengo.wikitty.*; +import org.sharengo.wikitty.solr.WikittySearchEngineSolr; /** * HBase must have 2 definitions, one for extension and one for wikitty object. @@ -65,7 +63,7 @@ public WikittyServiceJDBC() { extensionStorage = new WikittyExtensionStorageJDBC(); wikittyStorage = new WikittyStorageJDBC(extensionStorage); - searchEngin = new WikittySearchEnginSolr(extensionStorage); + searchEngin = new WikittySearchEngineSolr(extensionStorage); } @Override Modified: wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml =================================================================== --- wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml 2009-11-04 10:18:51 UTC (rev 307) +++ wikengo_core-wikitty/wikengo_core-wikitty-jdbc-impl/src/main/resources/solrconfig.xml 2009-11-04 10:19:47 UTC (rev 308) @@ -678,7 +678,7 @@ </queryResponseWriter> - <queryParser name="wikitty" class="org.sharengo.wikitty.jdbc.WikittyQueryParser"/> + <queryParser name="wikitty" class="org.sharengo.wikitty.solr.WikittyQueryParser"/>  <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter"> <int name="xsltCacheLifetimeSeconds">5</int> - </queryResponseWriter> + </queryResponseWriter> + <queryParser name="wikitty" class="org.sharengo.wikitty.solr.WikittyQueryParser"/> +

1 0