- Wikitty-commits - mailman.cloud.codelutin.com

r596 - trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 19:06:23 +0100 (Fri, 10 Dec 2010) New Revision: 596 Url: http://nuiton.org/repositories/revision/wikitty/596 Log: use Appconfig.getOptionAsList method Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java =================================================================== --- trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java 2010-12-10 18:05:38 UTC (rev 595) +++ trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java 2010-12-10 18:06:23 UTC (rev 596) @@ -35,6 +35,7 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; @@ -181,11 +182,10 @@ Properties result = null; // list of all properties file to load - String wikittyQueryFileList = config.getOption( - WikittyConfig.WikittyOption.WIKITTY_STORAGE_JDBC_QUERY_FILE.getKey()); + List<String> wikittyQueryFiles = config.getOptionAsList( + WikittyConfig.WikittyOption.WIKITTY_STORAGE_JDBC_QUERY_FILE.getKey()) + .getOption(); - String[] wikittyQueryFiles = wikittyQueryFileList.split(","); - for (String file : wikittyQueryFiles) { // create new Properties with result as parent result = new Properties(result);

1 0

r595 - in trunk/wikitty-api/src/main: java/org/nuiton/wikitty java/org/nuiton/wikitty/services xmi
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 19:05:38 +0100 (Fri, 10 Dec 2010) New Revision: 595 Url: http://nuiton.org/repositories/revision/wikitty/595 Log: - add Date in WikittyToken - add Date in WikittyAccessStat - WikittyAccessStat refactore storage to support monitoring with log(file), wikitty(db) or other Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java trunk/wikitty-api/src/main/xmi/wikitty.properties trunk/wikitty-api/src/main/xmi/wikitty.zargo Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-10 18:03:19 UTC (rev 594) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-10 18:05:38 UTC (rev 595) @@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; import org.nuiton.util.ArgumentsParserException; +import org.nuiton.wikitty.services.WikittyServiceAccessStat.AccessStatStorageWikitty; /** * Configuration de tous les modules Wikitty. La configuration par defaut doit @@ -222,6 +223,16 @@ + " or org.nuiton.wikitty.services.WikittyServiceInMemoryJdbcSolr)"), WikittyServiceInMemory.class.getName(), String.class, false, false), + WIKITTY_WIKITTYSERICEACCESSSTAT_COMPONENTS( + "wikitty.WikittyServiceAccessStat.components", + _("Indique le composant a utiliser pour le stockage des acces"), + AccessStatStorageWikitty.class.getName(), String.class, false, false), + + WIKITTY_ACCESSSTAT_EXTENSIONS( + "wikitty.service.accessstat.extensions", + _("Indique la liste d'extension dont il faut monitorer l'acces"), + "WikittyPubText,WikittyPubData", String.class, false, false), + WIKITTY_SERVER_CONFIG( "wikitty.service.server.config", _("Jetty server configuration file name"), @@ -250,11 +261,6 @@ _("Indique si le cache retourne des copies des objets ou des proxies"), "false", Boolean.class, false, false), - WIKITTY_ACCESSSTAT_EXTENSIONS( - "wikitty.service.accessstat.extensions", - _("Indique la liste d'extension dont il faut monitorer l'acces"), - "WikittyPubText,WikittyPubData", String.class, false, false), - WIKITTY_EVENT_PROPAGATE( "wikitty.service.event.propagate", _("Indique si le service d'event propage sur le reseau les evenements"), Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java 2010-12-10 18:03:19 UTC (rev 594) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java 2010-12-10 18:05:38 UTC (rev 595) @@ -4,13 +4,13 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.StringTokenizer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; @@ -19,6 +19,8 @@ import org.nuiton.wikitty.WikittyTree; import org.nuiton.wikitty.entities.BusinessEntity; import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyAccessStat; +import org.nuiton.wikitty.entities.WikittyAccessStatHelper; import org.nuiton.wikitty.entities.WikittyAccessStatImpl; import org.nuiton.wikitty.entities.WikittyTokenHelper; import org.nuiton.wikitty.entities.WikittyTreeNode; @@ -37,16 +39,60 @@ /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(WikittyServiceAccessStat.class); - Set<String> extensions = new HashSet<String>(); + static public interface AccessStatStorage { + /** + * @param ws service that ask stockage + * @param securityToken + * @param stat WikittyAccessStat wikitty + */ + public void store(WikittyServiceAccessStat ws, String securityToken, + Collection<Wikitty> stats); + } - public WikittyServiceAccessStat(ApplicationConfig config, WikittyService service) { + /** store access in wikitty */ + static public class AccessStatStorageWikitty implements AccessStatStorage { + @Override + public void store(WikittyServiceAccessStat ws, String securityToken, + Collection<Wikitty> stats) { + ws.getDelegate().store(securityToken, stats, false); + } + } + /** store access via common-logging, you must configure it to store + "AccessStat" logger correctly for info level */ + static public class AccessStatStorageLog implements AccessStatStorage { + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog("AccessStat"); + @Override + public void store(WikittyServiceAccessStat ws, String securityToken, + Collection<Wikitty> stats) { + for(Wikitty w : stats) { + Date date = WikittyAccessStatHelper.getDate(w); + String token = WikittyAccessStatHelper.getToken(w); + String user = WikittyAccessStatHelper.getUser(w); + String restored = WikittyAccessStatHelper.getRestored(w); + log.info(String.format("[%1$tF %1$tT] in session '%s' user '%s' restore '%s'", date, token, user, restored)); + } + } + } + + protected AccessStatStorage statStorage; + protected Set<String> extensions = new HashSet<String>(); + + public WikittyServiceAccessStat(ApplicationConfig config, WikittyService service, + AccessStatStorage statStorage) { super(service); - List<String> exts = config.getOptionAsList(WikittyConfig.WikittyOption. - WIKITTY_ACCESSSTAT_EXTENSIONS.getKey()).getOption(); - if (log.isInfoEnabled()) { - log.info(String.format("Monitor access to extensions %s", exts)); + this.statStorage = statStorage; + if (statStorage == null) { + log.warn("No AccessStatStorage available, access stat can't work"); + } else { + // if no statStorage, make extensions to never call statStorage + List<String> exts = config.getOptionAsList(WikittyConfig.WikittyOption. + WIKITTY_ACCESSSTAT_EXTENSIONS.getKey()).getOption(); + if (log.isInfoEnabled()) { + log.info(String.format("Monitor access to extensions %s", exts)); + } + extensions.addAll(exts); } - extensions.addAll(exts); } /** @@ -123,6 +169,7 @@ } WikittyAccessStatImpl stat = new WikittyAccessStatImpl(); + stat.setDate(new Date()); stat.setToken(securityToken); stat.setUser(user); stat.setRestored(id); @@ -131,7 +178,7 @@ } } - getDelegate().store(securityToken, stats, false); + statStorage.store(this, securityToken, stats); } @Override Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java 2010-12-10 18:03:19 UTC (rev 594) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceSecurity.java 2010-12-10 18:05:38 UTC (rev 595) @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -109,6 +110,7 @@ // force add extension to wikitty WikittyTokenHelper.addExtension(wikittyToken); WikittyTokenHelper.setUser(wikittyToken, user.getId()); + WikittyTokenHelper.setDate(wikittyToken, new Date()); ws.store(null, Arrays.asList(wikittyToken), false); log.debug(String.format("token '%s' is for login '%s'", tokenId, login)); Modified: trunk/wikitty-api/src/main/xmi/wikitty.properties =================================================================== --- trunk/wikitty-api/src/main/xmi/wikitty.properties 2010-12-10 18:03:19 UTC (rev 594) +++ trunk/wikitty-api/src/main/xmi/wikitty.properties 2010-12-10 18:05:38 UTC (rev 595) @@ -25,7 +25,7 @@ org.nuiton.wikitty.entities.WikittyAuthorisation.class.tagvalue.version=1.0 org.nuiton.wikitty.entities.WikittyGroup.class.tagvalue.version=1.0 org.nuiton.wikitty.entities.WikittyLabel.class.tagvalue.version=1.0 -org.nuiton.wikitty.entities.WikittyToken.class.tagvalue.version=1.0 +org.nuiton.wikitty.entities.WikittyToken.class.tagvalue.version=2.0 org.nuiton.wikitty.entities.WikittyTreeNode.class.tagvalue.version=2.0 org.nuiton.wikitty.entities.WikittyUser.class.tagvalue.version=1.0 org.nuiton.wikitty.entities.WikittyI18n.class.tagvalue.version=1.0 Modified: trunk/wikitty-api/src/main/xmi/wikitty.zargo =================================================================== (Binary files differ)

1 0

r594 - trunk/wikitty-api/src/main/java/org/nuiton/wikitty
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 19:03:19 +0100 (Fri, 10 Dec 2010) New Revision: 594 Url: http://nuiton.org/repositories/revision/wikitty/594 Log: use new AppConfig.getOptionAsList method Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyServiceFactory.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyServiceFactory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyServiceFactory.java 2010-12-10 12:22:57 UTC (rev 593) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyServiceFactory.java 2010-12-10 18:03:19 UTC (rev 594) @@ -123,24 +123,25 @@ static protected List<Class> getComponents(ApplicationConfig config, String prefix, Class clazz) { - List<Class> result = null; String key = prefix + clazz.getSimpleName() + ".components"; + List<Class> result = config.getOptionAsList(key).getOptionAsClass(); - String componentsString = config.getOption(key); - if (componentsString != null) { - String[] componentsList = componentsString.split(","); - result = new ArrayList<Class>(); - for(String className : componentsList) { - try { - Class c = Class.forName(className); - result.add(c); - } catch (ClassNotFoundException eee) { - throw new WikittyException(String.format( - "Can't find class %s, check your configuration and" - + " your jar file available", className), eee); - } - } - } + +// String componentsString = config.getOption(key); +// if (componentsString != null) { +// String[] componentsList = componentsString.split(","); +// result = new ArrayList<Class>(); +// for(String className : componentsList) { +// try { +// Class c = Class.forName(className); +// result.add(c); +// } catch (ClassNotFoundException eee) { +// throw new WikittyException(String.format( +// "Can't find class %s, check your configuration and" +// + " your jar file available", className), eee); +// } +// } +// } return result; } }

1 0

r593 - in trunk: wikitty-dto wikitty-perf-test wikitty-publication wikitty-ui-gwt
by tchemit＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: tchemit Date: 2010-12-10 13:22:57 +0100 (Fri, 10 Dec 2010) New Revision: 593 Url: http://nuiton.org/repositories/revision/wikitty/593 Log: fix svn:ignore Modified: trunk/wikitty-dto/ trunk/wikitty-perf-test/ trunk/wikitty-publication/ trunk/wikitty-ui-gwt/ Property changes on: trunk/wikitty-dto ___________________________________________________________________ Modified: svn:ignore - target PutObjectStoreDirHere *.iml *.iws *.ipr + PutObjectStoreDirHere target *.iml *.ipr *.iws .classpath .project .settings *.log Property changes on: trunk/wikitty-perf-test ___________________________________________________________________ Added: svn:ignore + target *.iml *.ipr *.iws .classpath .project .settings Property changes on: trunk/wikitty-publication ___________________________________________________________________ Modified: svn:ignore - target + target *.iml *.ipr *.iws .classpath .project .settings Property changes on: trunk/wikitty-ui-gwt ___________________________________________________________________ Added: svn:ignore + target *.iml *.ipr *.iws .classpath .project .settings *.log

1 0

r592 - in trunk: . wikitty-publication wikitty-publication/src/main/java/org/nuiton/wikitty/publication wikitty-publication/src/main/resources wikitty-publication/src/main/webapp/WEB-INF/jsp wikitty-publication/src/main/webapp/css wikitty-publication/src/main/xmi wikitty-publication/src/site/rst
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 05:50:09 +0100 (Fri, 10 Dec 2010) New Revision: 592 Url: http://nuiton.org/repositories/revision/wikitty/592 Log: Evolution #1134: new wikitty-publication module - add upload file for binary type - simplified url - add more object in eval script context Added: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractActionOnWikitty.java Modified: trunk/pom.xml trunk/wikitty-publication/ trunk/wikitty-publication/pom.xml trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractAction.java trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEdit.java trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEval.java trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionRaw.java trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionView.java trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/WikittyPublicationContext.java trunk/wikitty-publication/src/main/resources/wikitty-publication-ws-default.properties trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/edit.jsp trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/view.jsp trunk/wikitty-publication/src/main/webapp/css/wikitty-publication.css trunk/wikitty-publication/src/main/xmi/wikitty-publication.zargo trunk/wikitty-publication/src/site/rst/wp-analyse.rst Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/pom.xml 2010-12-10 04:50:09 UTC (rev 592) @@ -44,6 +44,13 @@ <version>2.3.8</version> </dependency> +  + <dependency> + <groupId>commons-fileupload</groupId> + <artifactId>commons-fileupload</artifactId> + <version>1.2.1</version> + </dependency> +  <dependency> <groupId>org.eclipse.jetty</groupId> Property changes on: trunk/wikitty-publication ___________________________________________________________________ Added: svn:ignore + target Modified: trunk/wikitty-publication/pom.xml =================================================================== --- trunk/wikitty-publication/pom.xml 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/pom.xml 2010-12-10 04:50:09 UTC (rev 592) @@ -67,6 +67,11 @@ <artifactId>commons-lang</artifactId> </dependency> + <dependency> + <groupId>commons-fileupload</groupId> + <artifactId>commons-fileupload</artifactId> + </dependency> + </dependencies>  Modified: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractAction.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractAction.java 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractAction.java 2010-12-10 04:50:09 UTC (rev 592) @@ -29,5 +29,4 @@ this.mapping = mapping; } - } Added: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractActionOnWikitty.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractActionOnWikitty.java (rev 0) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/AbstractActionOnWikitty.java 2010-12-10 04:50:09 UTC (rev 592) @@ -0,0 +1,152 @@ +package org.nuiton.wikitty.publication; + + +import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.wikitty.WikittyUtil; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyExtension; +import org.nuiton.wikitty.publication.entities.WikittyPubData; +import org.nuiton.wikitty.publication.entities.WikittyPubDataHelper; +import org.nuiton.wikitty.publication.entities.WikittyPubText; +import org.nuiton.wikitty.publication.entities.WikittyPubTextHelper; +import org.nuiton.wikitty.search.Criteria; +import org.nuiton.wikitty.search.Search; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public abstract class AbstractActionOnWikitty extends AbstractAction { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(AbstractActionOnWikitty.class); + + static final public int ARG_QUERY = 0; + static final public String ARG_MIMETYPE = "mimetype"; + static final public String ARG_CONTENT_FIELD = "contentField"; + static final public String SEARCH_SEPARATOR = ":"; + + abstract protected String getExampleUsage(); + + /** + * Recherche le type mime dont on a besoin. + * Par defaut recherche dans les arguments si on a dans l'ordre de preference + * <li> ARG_MIMETYPE + SEARCH_SEPARATOR + name + * (ex: mimetype:MyCommand.name:df=text/plain) + * <li> ARG_MIMETYPE + * (ex: mimetype=MyCommand.mimetype) + * <li> si w a l'extension WikittyPubText alors on prend la valeur du champs mimetype + * <li> si w a l'extension WikittyPubData alors on prend la valeur du champs mimetype + * <li> on retourne null + * + * @param context + * @param name la chaine utilise pour faire la recherche du wikitty + * @param w le wikitty trouve grace a name + * @return le mimetype ou null + */ + protected String getMimeType(WikittyPublicationContext context, + String name, Wikitty w) { + // looking for mimetype field + String mimetype = context.getArgument( + ARG_MIMETYPE + SEARCH_SEPARATOR + name, null); + if (mimetype == null) { + mimetype = context.getArgument(ARG_MIMETYPE, null); + } + if (mimetype != null) { + int i = mimetype.indexOf(WikittyUtil.FQ_FIELD_NAME_SEPARATOR); + if (i > 0) { // perhaps fully qualified field + String extName = WikittyExtension.extractExtensionName(mimetype); + String fieldName = WikittyExtension.extractFieldName(mimetype); + if (w.hasField(extName, fieldName)) { + // mimetype target field in wikitty + // replace with field value + mimetype = w.getFieldAsString(extName, fieldName); + } + } + } else if (w.hasExtension(WikittyPubText.EXT_WIKITTYPUBTEXT)) { + mimetype = WikittyPubTextHelper.getMimeType(w); + } else if (w.hasExtension(WikittyPubData.EXT_WIKITTYPUBDATA)) { + mimetype = WikittyPubDataHelper.getMimeType(w); + } + return mimetype; + } + + /** + * Retourne le critere pour recherche l'objet sur lequel faire l'action. + * Le nom du critere doit etre convenablement positionné avec la chaine + * qui a permit la recherche + * + * @param context + * @return + */ + protected Criteria searchCriteria(List<String> subContext) { + Criteria result; + if (subContext.size() <= 0) { + result = null; + } else { + String searchString = subContext.get(ARG_QUERY); + if (searchString.contains(SEARCH_SEPARATOR)) { + // on a un field=value + String[] arg = searchString.split(SEARCH_SEPARATOR); + result = Search.query().eq(arg[0], arg[1]).criteria(searchString); + } else { + // on a pas le champs, alors par defaut on recherche dans + // WikittyPubText.name et WikittyPubData.name + result = Search.query().or() + .eq(WikittyPubText.FQ_FIELD_WIKITTYPUBTEXT_NAME, searchString) + .eq(WikittyPubData.FQ_FIELD_WIKITTYPUBDATA_NAME, searchString) + .criteria(searchString); + } + } + return result; + } + + /** + * Recherche le champs contenant le content dont on a besoin. + * Par defaut recherche dans les arguments si on a dans l'ordre de preference + * <li> ARG_CONTENT_FIELD + SEARCH_SEPARATOR + name + * (ex: contentField:MyCommand.name:df=MyCommand.script) + * <li> ARG_CONTENT_FIELD + * (ex: contentField=MyCommand.script) + * <li> si w a l'extension WikittyPubText alors on prend le champs content + * <li> si w a l'extension WikittyPubData alors on prend le champs content + * <li> on retourne null + * + * @param context + * @param name la chaine utilise pour faire la recherche du wikitty + * @param w le wikitty trouve grace a name + * @return le champs contenant le content ou null + */ + protected String getContentFieldName(WikittyPublicationContext context, + String name, Wikitty w) { + String contentField = context.getArgument( + ARG_CONTENT_FIELD + SEARCH_SEPARATOR + name, null); + if (contentField == null) { + contentField = context.getArgument(ARG_CONTENT_FIELD, null); + } + if (contentField == null) { + if (w.hasExtension(WikittyPubText.EXT_WIKITTYPUBTEXT)) { + contentField = WikittyPubText.FQ_FIELD_WIKITTYPUBTEXT_CONTENT; + } else if (w.hasExtension(WikittyPubData.EXT_WIKITTYPUBDATA)) { + contentField = WikittyPubData.FQ_FIELD_WIKITTYPUBDATA_CONTENT; + } + } + return contentField; + } + + protected String getError(WikittyPublicationContext context) { + context.setContentType("text/html"); + String result = String.format( + "<h1>bad query %s</h1>" + + "Usage example" + + "<pre>%s</pre>", context.getRequest().getPathInfo(), getExampleUsage()); + return result; + } + +} Modified: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEdit.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEdit.java 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEdit.java 2010-12-10 04:50:09 UTC (rev 592) @@ -1,15 +1,21 @@ package org.nuiton.wikitty.publication; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.disk.DiskFileItemFactory; +import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.WikittyProxy; +import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.entities.WikittyImpl; import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.Search; import org.nuiton.wikitty.search.operators.Element; /** @@ -20,11 +26,17 @@ * Last update: $Date$ * by : $Author$ */ -public class ActionEdit extends AbstractAction { +public class ActionEdit extends AbstractActionOnWikitty { /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(ActionError.class); + static final protected String exampleUsage = + "edit/WikittyPubText.name=Wiki\n" + + "edit/Tuto\n" + + "edit/WikittyPubText.name=Tuto\n" + + "edit/Command.id=df\n"; + protected ApplicationConfig appConfig; public ActionEdit(ApplicationConfig appConfig) { @@ -33,33 +45,29 @@ @Override public Object doAction(WikittyPublicationContext context) { + String result; + Wikitty w; + WikittyProxy proxy = context.getWikittyProxy(); - String result = "no query"; - if (context.getMandatoryArguments().size() > 0) { - log.info("args " + context.getMandatoryArguments()); - String [] arg = context.getMandatoryArguments().get(0).split("="); - - Wikitty w; - switch (arg.length) { - case 0: - // new wikitty without extension and edit - w = new WikittyImpl(); - break; - case 1: - // new wikitty with one extension and edit - w = new WikittyImpl(); - WikittyExtension ext = proxy.restoreExtensionLastVersion(arg[0]); - if (ext != null) { - w.addExtension(ext); - } - break; - default: - // find and edit - Criteria criteria = Search.query().eq(arg[0], arg[1]).criteria(); - w = proxy.findByCriteria(criteria); + if (context.getArguments().containsKey("delete")) { + // on nous demande supprimer le wikitty, on l'efface et on + // affichera un wikitty vide + String id = context.getArguments().get("id"); + proxy.delete(id); + // apres un effacement on reprend l'edition d'un tout nouveau wikitty + w = new WikittyImpl(); + } else { + // recherche du Wikitty a editer ou creation d'un nouveau si necessaire + Criteria criteria = searchCriteria(context.getMandatoryArguments()); + if (criteria == null) { + w = new WikittyImpl(); + } else { + w = proxy.findByCriteria(criteria); } + // si on ne retrouve pas le wikitty, mais qu'il vient d'etre cree + // pour l'edition, on recree un wikitty avec ce meme identifiant if (w == null && "0.0".equals(context.getArguments().get("version"))) { // c'est un nouvel objet, il n'a pas encore ete sauve, mais on veut le faire String id = context.getArguments().get("id"); @@ -67,51 +75,106 @@ } if (w == null) { - context.setContentType("text/plain"); - result = String.format( - "no data found for %s with value %s", arg[0], arg[1]); + // si le wikitty est null, et qu'on etait pas en edition + // cela signifie qu'on ne retrouve pas le wikitty a editer + // on creer un nouveau wikitty vide que l'on editera + w = new WikittyImpl(); } else { - if (context.getArguments().containsKey("delete")) { - String id = context.getArguments().get("id"); - context.getWikittyProxy().delete(id); - } else if (context.getArguments().containsKey("store")) { - // change and store wikitty + // on met a jour le wikitty avec les infos trouvees dans les + // arguments - // ajout des extensions si necessaire - String [] extNames = context.getRequest().getParameterValues("extensions"); - if (extNames != null) { - for (String extName : extNames) { - WikittyExtension ext = proxy.restoreExtensionLastVersion(extName); - if (ext != null) { - w.addExtension(ext); - } - } +// // ajout des extensions si necessaire +// String[] extNames = context.getRequest().getParameterValues("extensions"); +// if (extNames != null) { +// for (String extName : extNames) { +// WikittyExtension ext = proxy.restoreExtensionLastVersion(extName); +// if (ext != null) { +// w.addExtension(ext); +// } +// } +// } + + // ajout des extensions si necessaire + String extName = context.getArgument("extensions", null); + if (extName != null) { + WikittyExtension ext = proxy.restoreExtensionLastVersion(extName); + if (ext != null) { + w.addExtension(ext); } + } - // modifie tous les champs presents dans le formulaire - for (Map.Entry<String, String> field : context.getArguments().entrySet()) { - if (field.getKey().contains(".")) { - String extName = WikittyExtension.extractExtensionName(field.getKey()); - String fieldName = WikittyExtension.extractFieldName(field.getKey()); +// boolean isMultipart = ServletFileUpload.isMultipartContent(context.getRequest()); +// if (isMultipart == true) { +// // Create a factory for disk-based file items +// DiskFileItemFactory factory = new DiskFileItemFactory(); +// // Create a new file upload handler +// ServletFileUpload upload = new ServletFileUpload(factory); +// // Process the uploaded items +// // Parse the request +// try { +// List<FileItem> items = upload.parseRequest(context.getRequest()); +// for (FileItem item : items) { +// String key = item.getFieldName(); +// if (key.contains(WikittyUtil.FQ_FIELD_NAME_SEPARATOR)) { +// String extName = WikittyExtension.extractExtensionName(key); +// String fieldName = WikittyExtension.extractFieldName(key); +// +// if (w.hasField(extName, fieldName)) { +// Object value = null; +// if (!"true".equals(context.getArgument("isNull-" + key, "false"))) { +// if (item.isFormField()) { +// value = item.getString(); +// } else { +// String filename = item.getName(); +// String mime = item.getContentType(); +// value = item.get(); +// log.info(String.format( +// "Set content with file '%s' of type '%s'", +// filename, mime)); +// } +// } +// w.setField(extName, fieldName, value); +// } +// } +// } +// } catch (FileUploadException eee) { +// log.error("Can't get uploaded file", eee); +// } +// } - if (w.hasField(extName, fieldName)) { - w.setField(extName, fieldName, field.getValue()); + + Map<String, Object> args = new HashMap<String, Object>(); + args.putAll(context.getArguments()); + args.putAll(context.getArgumentFiles()); + for (Map.Entry<String, Object> field : args.entrySet()) { + String key = field.getKey(); + Object value = null; + if (key.contains(WikittyUtil.FQ_FIELD_NAME_SEPARATOR)) { + String ext = WikittyExtension.extractExtensionName(key); + String fieldName = WikittyExtension.extractFieldName(key); + + if (w.hasField(ext, fieldName)) { + if (!"true".equals(context.getArgument("isNull-" + key, "false"))) { + value = field.getValue(); } + w.setField(ext, fieldName, value); } } + } - // sauve le wikitty modifie - context.getWikittyProxy().store(w); + if (context.getArguments().containsKey("store")) { + // on nous demande la sauvegarde + proxy.store(w); } - - // forward l'affichage de la page vers la jsp d'edition - context.setContentType("forward/jsp"); - context.getRequest().setAttribute( - WikittyPublicationContext.CONTEXT_VAR, context); - context.getRequest().setAttribute("wikitty", w); - result = "/WEB-INF/jsp/edit.jsp"; } } + + // forward l'affichage de la page vers la jsp d'edition + context.setContentType("forward/jsp"); + context.getRequest().setAttribute( + ActionEval.CONTEXT_VAR, context); + context.getRequest().setAttribute(ActionEval.WIKITTY_VAR, w); + result = "/WEB-INF/jsp/edit.jsp"; return result; } @@ -126,9 +189,14 @@ */ public String makeUrl(WikittyPublicationContext context, Wikitty w) { String url = getMapping() + "/" - + Element.ELT_ID + "=" + w.getId(); + + Element.ELT_ID + SEARCH_SEPARATOR + w.getId(); url = context.makeUrl(url); return url; } + @Override + protected String getExampleUsage() { + return exampleUsage; + } + } Modified: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEval.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEval.java 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionEval.java 2010-12-10 04:50:09 UTC (rev 592) @@ -1,6 +1,9 @@ package org.nuiton.wikitty.publication; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.script.Bindings; import javax.script.ScriptEngine; import javax.script.ScriptEngineFactory; @@ -11,9 +14,10 @@ import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.WikittyException; import org.nuiton.wikitty.WikittyProxy; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.publication.entities.WikittyPubText; import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.Search; /** * Permet d'evaluer un WikittyPubText et de retourner la valeur de l'evaluation. @@ -34,11 +38,35 @@ * Last update: $Date$ * by : $Author$ */ -public class ActionEval extends AbstractAction { +public class ActionEval extends AbstractActionOnWikitty { /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(ActionError.class); + /** variable contenant l'instance de la classe ActionEval */ + static final public String EVAL_VAR = "wpEval"; + /** variable name use to put context in script and jsp */ + static final public String CONTEXT_VAR = "wpContext"; + /** contient la liste des arguments mandatory non encore utilise */ + static final public String SUBCONTEXT_VAR = "wpSubContext"; + /** + * contient le nom de la page WikittyPubText (ex: Wiki) ou la requete ayant + * permis de trouver la page (ex: MyScript.name=df) + */ + static final public String PAGE_NAME_VAR = "wpPage"; + /** + * contient le wikitty utilise comme script + */ + static final public String WIKITTY_VAR = "wpWikitty"; + + static final protected String exampleUsage = + "eval/WikittyPubText.name=Wiki/WikittyPubText.content?mimetype=WikittyPubText.mimetype\n" + + "eval/Tuto\n" + + "eval/Tuto/WikittyPubText.content\n" + + "eval/WikittyPubText.name=Tuto/WikittyPubText.content\n" + + "eval/WikittyPubText.name=Tuto/WikittyPubText.content?mimetype=WikittyPubText.mimetype\n" + + "eval/Command.id=df/Command.script?mimetype=Command.mimetype\n"; + protected ApplicationConfig appConfig; protected ScriptEngineManager scriptEnginManager; @@ -48,52 +76,99 @@ scriptEnginManager = new ScriptEngineManager(); } - @Override - public Object doAction(WikittyPublicationContext context) { - WikittyProxy proxy = context.getWikittyProxy(); + protected Object eval(String name, String script, String mimetype, + Map<String, Object> bindings) { + ScriptEngine scriptEngin = scriptEnginManager.getEngineByMimeType(mimetype); + if (scriptEngin == null) { + List<ScriptEngineFactory> factories = + scriptEnginManager.getEngineFactories(); + String msgFactories = ""; + for (ScriptEngineFactory f : factories) { + msgFactories += String.format( + "\n%s extensions: %s mimetypes: %s", + f.getEngineName(), f.getExtensions(), f.getMimeTypes()); + } + throw new WikittyException(String.format( + "Can't find engine for %s(%s). Available engines: %s", + name, mimetype, msgFactories)); + } else { + Bindings b = scriptEngin.createBindings(); + b.putAll(bindings); + try { + Object result = scriptEngin.eval(script, b); + return result; + } catch (ScriptException eee) { + throw new WikittyException(String.format( + "Can't evaluated script %s(%s=>%s) script was\n%s", + name, mimetype, + scriptEngin.getFactory().getEngineName(), script), eee); + } + } - Object result = "no query"; - if (context.getMandatoryArguments().size() > 0) { - log.info("args " + context.getMandatoryArguments()); - String [] arg = context.getMandatoryArguments().get(0).split("="); + } - Criteria criteria = Search.query().eq(arg[0], arg[1]).criteria(); - WikittyPubText w = proxy.findByCriteria(WikittyPubText.class, criteria); + public Object doAction(WikittyPublicationContext context, List<String> subContext) { + log.info("path " + subContext); + + Object result; + + Criteria criteria = searchCriteria(subContext); + + if (criteria == null) { + // rien a evaluer, on retourne une chaine vide + result = ""; + } else { + WikittyProxy proxy = context.getWikittyProxy(); + Wikitty w = proxy.findByCriteria(criteria); + if (w == null) { + context.setContentType("text/plain"); result = String.format( - "no data found for %s with value %s", arg[0], arg[1]); + "no data found for criteria %s", criteria); } else { - String mimetype = w.getMimeType(); - ScriptEngine scriptEngin = scriptEnginManager.getEngineByMimeType(mimetype); - if (scriptEngin == null) { - List<ScriptEngineFactory> factories = - scriptEnginManager.getEngineFactories(); - String msgFactories = ""; - for (ScriptEngineFactory f : factories) { - msgFactories += String.format( - "\n%s extensions: %s mimetypes: %s", - f.getEngineName(), f.getExtensions(), f.getMimeTypes()); - } - throw new WikittyException(String.format( - "Can't find engine for %s(%s). Available engines: %s", - w.getName(), w.getMimeType(), msgFactories)); + String contentField = getContentFieldName(context, criteria.getName(), w); + + if (contentField == null) { + result = getError(context); } else { - Bindings bindings = scriptEngin.createBindings(); - bindings.put(WikittyPublicationContext.CONTEXT_VAR, context); - String script = w.getContent(); - try { - result = scriptEngin.eval(script, bindings); - } catch (ScriptException eee) { - throw new WikittyException(String.format( - "Can't evaluated script %s(%s=>%s) script was\n%s", - w.getName(), w.getMimeType(), - scriptEngin.getFactory().getEngineName(), script), eee); - } + String extName = WikittyExtension.extractExtensionName(contentField); + String fieldName = WikittyExtension.extractFieldName(contentField); + + String mimetype = getMimeType(context, criteria.getName(), w); + String content = w.getFieldAsString(extName, fieldName); + + // supprime de subcontext ce qui a ete utilise dans cette methode + subContext = new ArrayList<String>(subContext.subList(1, subContext.size())); + + Map<String, Object> bindings = new HashMap<String, Object>(); + bindings.put(PAGE_NAME_VAR, criteria.getName()); + bindings.put(CONTEXT_VAR, context); + bindings.put(SUBCONTEXT_VAR, subContext); + bindings.put(WIKITTY_VAR, w); + bindings.put(EVAL_VAR, this); + + result = eval(criteria.getName(), content, mimetype, bindings); } } } + return result; + } + + @Override + public Object doAction(WikittyPublicationContext context) { + Object result; + if (context.getMandatoryArguments().size() <= 0) { + result = getError(context); + } else { + result = doAction(context, context.getMandatoryArguments()); + } return result; } + @Override + protected String getExampleUsage() { + return exampleUsage; + } + } Modified: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionRaw.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionRaw.java 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionRaw.java 2010-12-10 04:50:09 UTC (rev 592) @@ -6,12 +6,7 @@ import org.nuiton.wikitty.WikittyProxy; import org.nuiton.wikitty.entities.Wikitty; import org.nuiton.wikitty.entities.WikittyExtension; -import org.nuiton.wikitty.publication.entities.WikittyPubData; -import org.nuiton.wikitty.publication.entities.WikittyPubDataHelper; -import org.nuiton.wikitty.publication.entities.WikittyPubText; -import org.nuiton.wikitty.publication.entities.WikittyPubTextHelper; import org.nuiton.wikitty.search.Criteria; -import org.nuiton.wikitty.search.Search; /** * Action permettant de retourner la valeur d'un champs d'un object. @@ -41,17 +36,16 @@ * Last update: $Date$ * by : $Author$ */ -public class ActionRaw extends AbstractAction { +public class ActionRaw extends AbstractActionOnWikitty { /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(ActionError.class); - static final public int ARG_QUERY = 0; - static final public int ARG_CONTENT_FIELD = 1; - static final public String ARG_MIMETYPE = "mimetype"; - static final protected String exampleUsage = "ram/WikittyPubData.name=MonImage.jpg/WikittyPubData.content?mimetype=WikittyPubData.mimetype\n" + + "ram/Tuto\n" + + "ram/Tuto/WikittyPubText.content\n" + + "ram/WikittyPubText.name=Tuto/WikittyPubText.content\n" + "ram/WikittyPubText.name=Tuto/WikittyPubText.content?mimetype=WikittyPubText.mimetype\n" + "ram/WikittyUser.login=admin/WikittyUser.password?mimetype=text/plain\n"; @@ -61,61 +55,41 @@ this.appConfig = appConfig; } + protected String getExampleUsage() { + return exampleUsage; + } + @Override public Object doAction(WikittyPublicationContext context) { - WikittyProxy proxy = context.getWikittyProxy(); - Object result = "no query"; - if (context.getMandatoryArguments().size() < 1 || - (context.getMandatoryArguments().size() < 2 - && !context.getMandatoryArguments().get(ARG_QUERY).startsWith("WikittyPub")) ) { - context.setContentType("text/html"); - result = String.format("<html><head></head><body>" - + "<h1>bad query</h1>" - + "Usage example" - + "<pre>%s</pre>" - + "</body></html>\n", exampleUsage); - } else { - log.info("args " + context.getMandatoryArguments()); - String [] arg = context.getMandatoryArguments().get(ARG_QUERY).split("="); + log.info("args " + context.getMandatoryArguments()); - Criteria criteria = Search.query().eq(arg[0], arg[1]).criteria(); + Object result; + Criteria criteria = searchCriteria(context.getMandatoryArguments()); + + if (criteria == null) { + result = getError(context); + } else { + WikittyProxy proxy = context.getWikittyProxy(); Wikitty w = proxy.findByCriteria(criteria); + if (w == null) { + context.setContentType("text/plain"); result = String.format( - "no data found for %s with value %s", arg[0], arg[1]); + "no data found for criteria %s", criteria); } else { - // looking for mimetype field - String mimetype = context.getArguments().get(ARG_MIMETYPE); - if (mimetype != null) { - int i = mimetype.indexOf('.'); - if (i > 0) { // perhaps fully qualified field - String extName = WikittyExtension.extractExtensionName(mimetype); - String fieldName = WikittyExtension.extractFieldName(mimetype); - if (w.hasField(extName, fieldName)) { - // mimetype target field in wikitty - // replace with field value - mimetype = w.getFieldAsString(extName, fieldName); - } - } + String contentField = getContentFieldName(context, criteria.getName(), w); + + if (contentField == null) { + result = getError(context); + } else { + String extName = WikittyExtension.extractExtensionName(contentField); + String fieldName = WikittyExtension.extractFieldName(contentField); + + String mimetype = getMimeType(context, criteria.getName(), w); context.setContentType(mimetype); - } else if (w.hasExtension(WikittyPubText.EXT_WIKITTYPUBTEXT)) { - mimetype = WikittyPubTextHelper.getMimeType(w); - } else if (w.hasExtension(WikittyPubData.EXT_WIKITTYPUBDATA)) { - mimetype = WikittyPubDataHelper.getMimeType(w); + result = w.getFieldAsObject(extName, fieldName); } - String contentField = null; - if (context.getMandatoryArguments().size() >= 2) { - contentField = context.getMandatoryArguments().get(ARG_CONTENT_FIELD); - } else if (w.hasExtension(WikittyPubText.EXT_WIKITTYPUBTEXT)) { - contentField = WikittyPubText.FQ_FIELD_WIKITTYPUBTEXT_CONTENT; - } else if (w.hasExtension(WikittyPubData.EXT_WIKITTYPUBDATA)) { - contentField = WikittyPubData.FQ_FIELD_WIKITTYPUBDATA_CONTENT; - } - String extName = WikittyExtension.extractExtensionName(contentField); - String fieldName = WikittyExtension.extractFieldName(contentField); - - result = w.getFieldAsObject(extName, fieldName); } } Modified: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionView.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionView.java 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/ActionView.java 2010-12-10 04:50:09 UTC (rev 592) @@ -1,14 +1,11 @@ package org.nuiton.wikitty.publication; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; import org.nuiton.util.StringUtil; import org.nuiton.wikitty.WikittyProxy; import org.nuiton.wikitty.entities.Wikitty; -import org.nuiton.wikitty.entities.WikittyExtension; -import org.nuiton.wikitty.entities.WikittyImpl; import org.nuiton.wikitty.search.Criteria; import org.nuiton.wikitty.search.PagedResult; import org.nuiton.wikitty.search.Search; @@ -57,12 +54,8 @@ // forward l'affichage de la page vers la jsp d'edition context.setContentType("forward/jsp"); - context.getRequest().setAttribute( - WikittyPublicationContext.CONTEXT_VAR, context); - context.getRequest().setAttribute("r", r); - context.getRequest().setAttribute("first", first); - context.getRequest().setAttribute("end", end); - context.getRequest().setAttribute("wikitty", w); + context.getRequest().setAttribute(ActionEval.CONTEXT_VAR, context); + context.getRequest().setAttribute(ActionEval.WIKITTY_VAR, w); context.getRequest().setAttribute("pagedResult", pagedResult); String result = "/WEB-INF/jsp/view.jsp"; Modified: trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/WikittyPublicationContext.java =================================================================== --- trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/WikittyPublicationContext.java 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/java/org/nuiton/wikitty/publication/WikittyPublicationContext.java 2010-12-10 04:50:09 UTC (rev 592) @@ -10,6 +10,10 @@ import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.disk.DiskFileItemFactory; +import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; @@ -21,6 +25,7 @@ import org.nuiton.wikitty.WikittyProxy; import org.nuiton.wikitty.WikittyService; import org.nuiton.wikitty.WikittyServiceFactory; +import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.entities.WikittyExtension; import org.nuiton.wikitty.entities.WikittyLabelAbstract; import org.nuiton.wikitty.entities.WikittyTreeNodeAbstract; @@ -41,9 +46,6 @@ /** to use log facility, just put in your code: log.info(\"...\"); */ static private Log log = LogFactory.getLog(WikittyPublicationContext.class); - /** variable name use to put context in script and jsp */ - static final public String CONTEXT_VAR = "WPContext"; - /** configuration option name FIXME poussin 20101206 use OptionDef */ static final public String CONFIG_FILE = "wikitty.publication.config.pattern"; static final public String ACTION_PREFIX = "wikitty.publication.action"; @@ -58,8 +60,10 @@ protected HttpServletResponse resp; protected String wsContext = null; protected String actionName = null; + protected String path = ""; protected List<String> mandatoryArguments = new ArrayList<String>(); protected Map<String, String> arguments = new HashMap<String, String>(); + protected Map<String, byte[]> argumentFiles = new HashMap<String, byte[]>(); protected WikittyProxy proxy = null; protected String contentType = "text/html"; @@ -115,14 +119,48 @@ if (comps.length > 1) { actionName = comps[2]; } - + + for (int i = 3; i < comps.length; i++) { mandatoryArguments.add(comps[i]); + path += "/" + comps[i]; } - for (Enumeration<String> e=req.getParameterNames(); e.hasMoreElements();) { - String name = e.nextElement(); - String value = req.getParameter(name); - arguments.put(name, value); + + + boolean isMultipart = ServletFileUpload.isMultipartContent(getRequest()); + if (isMultipart == true) { + // Create a factory for disk-based file items + DiskFileItemFactory factory = new DiskFileItemFactory(); + // Create a new file upload handler + ServletFileUpload upload = new ServletFileUpload(factory); + // Process the uploaded items + // Parse the request + try { + List<FileItem> items = upload.parseRequest(getRequest()); + for (FileItem item : items) { + String name = item.getFieldName(); + if (item.isFormField()) { + String value = item.getString(); + arguments.put(name, value); + } else { + String filename = item.getName(); + String mime = item.getContentType(); + byte[] value = item.get(); + log.info(String.format( + "Argument file '%s' of type '%s'", + filename, mime)); + argumentFiles.put(name, value); + } + } + } catch (FileUploadException eee) { + log.error("Can't get uploaded file", eee); + } + } else { + for (Enumeration<String> e = req.getParameterNames(); e.hasMoreElements();) { + String name = e.nextElement(); + String value = req.getParameter(name); + arguments.put(name, value); + } } log.info(String.format( "path %s => ws: %s action: %s mandatoryArguments: %s arguments: %s", @@ -141,10 +179,23 @@ return appConfig; } + /** + * le nom de l'action a faire + * @return + */ public String getActionName() { return actionName; } + /** + * le reste de l'url apres l'action + * @return + */ + public String getPath() { + return path; + } + + public WikittyProxy getWikittyProxy() { if (proxy == null) { proxy = new WikittyProxy(getAppConfig(), getWikittyService()); @@ -223,6 +274,10 @@ return arguments; } + public Map<String, byte[]> getArgumentFiles() { + return argumentFiles; + } + public String getArgument(String name, String defaultValue) { String result = defaultValue; if (getArguments().containsKey(name)) { Modified: trunk/wikitty-publication/src/main/resources/wikitty-publication-ws-default.properties =================================================================== --- trunk/wikitty-publication/src/main/resources/wikitty-publication-ws-default.properties 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/resources/wikitty-publication-ws-default.properties 2010-12-10 04:50:09 UTC (rev 592) @@ -37,7 +37,8 @@ wikitty.WikittyService.components=org.nuiton.wikitty.services.WikittyServiceStorage,\ org.nuiton.wikitty.services.WikittyServiceNotifier,\ org.nuiton.wikitty.services.WikittyServiceCached,\ -org.nuiton.wikitty.services.WikittyServiceSecurity +org.nuiton.wikitty.services.WikittyServiceSecurity,\ +org.nuiton.wikitty.services.WikittyServiceAccessStat wikitty.WikittyServiceStorage.components=org.nuiton.wikitty.jdbc.WikittyExtensionStorageJDBC,\ org.nuiton.wikitty.jdbc.WikittyStorageJDBC,\ org.nuiton.wikitty.solr.WikittySearchEngineSolr @@ -45,6 +46,7 @@ wikitty.service.cache.allwaysRestoreCopies=false wikitty.service.event.propagate=false wikitty.service.event.listen=false +wikitty.service.accessstat.extensions=WikittyPubText,WikittyPubData wikitty.addon.export.threadnumber=1 wikitty.addon.export.directory=${wikitty.data.directory}/export wikitty.addon.export.publicurl=file://${wikitty.data.directory}/export/ Modified: trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/edit.jsp =================================================================== --- trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/edit.jsp 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/edit.jsp 2010-12-10 04:50:09 UTC (rev 592) @@ -4,6 +4,7 @@ Author : poussin --%> +<%@page import="org.nuiton.wikitty.publication.ActionEval"%> <%@page import="org.apache.commons.lang.StringEscapeUtils"%> <%@page import="org.nuiton.wikitty.entities.FieldType"%> <%@page import="java.util.Collection"%> @@ -14,21 +15,22 @@ <%@page import="org.nuiton.wikitty.entities.Wikitty"%> <%@page import="org.nuiton.wikitty.publication.WikittyPublicationContext"%> -<h1>Edit ${wikitty.id}</h1> - -<pre>${wikitty}</pre> - <% WikittyPublicationContext context = (WikittyPublicationContext) - request.getAttribute(WikittyPublicationContext.CONTEXT_VAR); -Wikitty wikitty = (Wikitty)request.getAttribute("wikitty"); + request.getAttribute(ActionEval.CONTEXT_VAR); +Wikitty wikitty = (Wikitty)request.getAttribute(ActionEval.WIKITTY_VAR); ActionEdit action = context.getAction(ActionEdit.class); String url = action.makeUrl(context, wikitty); %> -<form action='<%=url%>' method="post"> - <input type="hidden" name="id" value="${wikitty.id}" /> - <input type="hidden" name="version" value="${wikitty.version}" /> +<h1>Edit <%=wikitty.getId()%></h1> + +<pre><%=wikitty%></pre> + +<form class="edit" action='<%=url%>' method="post" enctype="multipart/form-data"> + <input type="hidden" name="MAX_FILE_SIZE" value="104857600" />  + <input type="hidden" name="id" value="<%=wikitty.getId()%>" /> + <input type="hidden" name="version" value="<%=wikitty.getVersion()%>" /> <fieldset> <legend><span class="legend">Extensions</span></legend> <% @@ -50,6 +52,7 @@ <% } %> + <input type="submit" name="addExtension" value="Add" /> </fieldset> <% @@ -67,6 +70,11 @@ <label for="<%=extName%>.<%=fieldName%>" tabindex="<%=++index%>"><span class="label"><%=fieldName%></span></label> <% switch(fieldType.getType()) { + case BINARY: + %> + <input type="file" name="<%=extName%>.<%=fieldName%>"/> + <% + break; case BOOLEAN: boolean valueBool = wikitty.getFieldAsBoolean(ext.getName(), fieldName); String checked = valueBool?"checked='true'":""; @@ -77,8 +85,11 @@ default: Object valueObject = wikitty.getFieldAsObject(ext.getName(), fieldName); String valueString = ""; + String checkedNull = ""; if (valueObject != null) { valueString = String.valueOf(valueObject); + } else { + checkedNull = "checked='true'"; } valueString = StringEscapeUtils.escapeHtml(valueString); @@ -91,6 +102,9 @@ <input type="text" name="<%=extName%>.<%=fieldName%>" value="<%=valueString%>" /> <% } + %> + <input type="checkbox" name="isNull-<%=extName%>.<%=fieldName%>" value="true" <%=checkedNull%>/>(null) + <% } %> <%=fieldType.toDefinition("")%> Modified: trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/view.jsp =================================================================== --- trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/view.jsp 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/webapp/WEB-INF/jsp/view.jsp 2010-12-10 04:50:09 UTC (rev 592) @@ -4,6 +4,7 @@ Author : poussin --%> +<%@page import="org.nuiton.wikitty.publication.ActionEval"%> <%@page import="org.nuiton.wikitty.publication.ActionEdit"%> <%@page import="org.nuiton.wikitty.publication.WikittyPublicationContext"%> <%@page import="org.nuiton.wikitty.search.PagedResult"%> @@ -13,23 +14,23 @@ <% WikittyPublicationContext context = (WikittyPublicationContext) - request.getAttribute(WikittyPublicationContext.CONTEXT_VAR); + request.getAttribute(ActionEval.CONTEXT_VAR); PagedResult<Wikitty> pagedResult = (PagedResult<Wikitty>)request.getAttribute("pagedResult"); -Wikitty wikitty = (Wikitty)request.getAttribute("wikitty"); +Wikitty wikitty = (Wikitty)request.getAttribute(ActionEval.WIKITTY_VAR); ActionEdit actionEdit = context.getAction(ActionEdit.class); %> <form action="" method="post"> <div> - <textarea name="r" rows="4" cols="20"><%=request.getAttribute("r")%></textarea> + <textarea name="r" rows="4" cols="20"><%=context.getArgument("r", "*")%></textarea> </div> <label for="first">First</label> <input type="text" name="first" value="<%=context.getArgument("first", "0")%>" /> <label for="first">First</label> <input type="text" name="end" value="<%=context.getArgument("end", "100")%>" /> <input type="submit" name="id" value="Search"/> <pre> -${wikitty} +<%=wikitty%> </pre> <%=pagedResult.getFirstIndice()%>-<%=pagedResult.getFirstIndice() + pagedResult.size()%>/${pagedResult.numFound} Modified: trunk/wikitty-publication/src/main/webapp/css/wikitty-publication.css =================================================================== --- trunk/wikitty-publication/src/main/webapp/css/wikitty-publication.css 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/main/webapp/css/wikitty-publication.css 2010-12-10 04:50:09 UTC (rev 592) @@ -1,4 +1,4 @@ -label { +.edit label { float : left; width : 200px; text-align: right; Modified: trunk/wikitty-publication/src/main/xmi/wikitty-publication.zargo =================================================================== (Binary files differ) Modified: trunk/wikitty-publication/src/site/rst/wp-analyse.rst =================================================================== --- trunk/wikitty-publication/src/site/rst/wp-analyse.rst 2010-12-10 04:45:01 UTC (rev 591) +++ trunk/wikitty-publication/src/site/rst/wp-analyse.rst 2010-12-10 04:50:09 UTC (rev 592) @@ -125,10 +125,10 @@ Par exemple les URLs suivantes sont équivalentes:: - /eval/WikittyPubText.name=Wiki + /eval/WikittyPubText.name:Wiki /eval/Wiki - /raw/WikittyPubText.name=WikiLogo + /raw/WikittyPubText.name:WikiLogo /raw/WikiLogo Action eval @@ -311,9 +311,9 @@ <html> ... - <div id="bandeau"><img src="<%=context.makeUrl("/raw/WikiLogo")%>"/></div> - <div id="menu"><%=eval.doAction(context, "/WikiMenu")%></div> - <div id="content"><%=eval.doAction(context, nextUrl!=null?nextUrl:"/WikiHello")%></div> + <div id="bandeau"><img src="<%=wpContext.makeUrl("/raw/WikiLogo")%>"/></div> + <div id="menu"><%=eval.doAction(wpContext, "/WikiMenu")%></div> + <div id="content"><%=eval.doAction(wpContext, wpSubContext")%></div> ... </html>

1 0

r591 - in trunk/wikitty-api/src/main: java/org/nuiton/wikitty java/org/nuiton/wikitty/services xmi
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 05:45:01 +0100 (Fri, 10 Dec 2010) New Revision: 591 Url: http://nuiton.org/repositories/revision/wikitty/591 Log: Evolution #1140: Add WikittyServiceAccessStat to make stat on wikitty restored Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java trunk/wikitty-api/src/main/xmi/wikitty.properties trunk/wikitty-api/src/main/xmi/wikitty.zargo Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-10 04:42:19 UTC (rev 590) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-10 04:45:01 UTC (rev 591) @@ -250,6 +250,11 @@ _("Indique si le cache retourne des copies des objets ou des proxies"), "false", Boolean.class, false, false), + WIKITTY_ACCESSSTAT_EXTENSIONS( + "wikitty.service.accessstat.extensions", + _("Indique la liste d'extension dont il faut monitorer l'acces"), + "WikittyPubText,WikittyPubData", String.class, false, false), + WIKITTY_EVENT_PROPAGATE( "wikitty.service.event.propagate", _("Indique si le service d'event propage sur le reseau les evenements"), Added: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java (rev 0) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceAccessStat.java 2010-12-10 04:45:01 UTC (rev 591) @@ -0,0 +1,187 @@ +package org.nuiton.wikitty.services; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.StringTokenizer; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.util.ApplicationConfig; +import org.nuiton.wikitty.WikittyConfig; +import org.nuiton.wikitty.WikittyService; +import org.nuiton.wikitty.WikittyTree; +import org.nuiton.wikitty.entities.BusinessEntity; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyAccessStatImpl; +import org.nuiton.wikitty.entities.WikittyTokenHelper; +import org.nuiton.wikitty.entities.WikittyTreeNode; +import org.nuiton.wikitty.search.Criteria; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class WikittyServiceAccessStat extends WikittyServiceDelegator { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(WikittyServiceAccessStat.class); + + Set<String> extensions = new HashSet<String>(); + + public WikittyServiceAccessStat(ApplicationConfig config, WikittyService service) { + super(service); + List<String> exts = config.getOptionAsList(WikittyConfig.WikittyOption. + WIKITTY_ACCESSSTAT_EXTENSIONS.getKey()).getOption(); + if (log.isInfoEnabled()) { + log.info(String.format("Monitor access to extensions %s", exts)); + } + extensions.addAll(exts); + } + + /** + * retourne l'id du user associe au token + * @param securityToken + * @return null si l'id du user n'a pas pu etre recuperer + */ + protected String getUserId(String securityToken) { + String result = null; + // recuperation de l'utilisateur associe au securityToken + // le securityToken est aussi l'id de l'objet + if (securityToken != null) { + Wikitty securityTokenWikitty = WikittyServiceEnhanced.restore( + getDelegate(), securityToken, securityToken); + if (securityTokenWikitty == null) { + // no exception, this service must never faild + log.warn("bad (obsolete ?) token"); + } else { + result = WikittyTokenHelper.getUser(securityTokenWikitty); + } + } + return result; + } + + /** + * Indique si dans la liste des extensions passees en parametre il y en + * a au moins une a surveiller + * + * @param exts + * @return vrai s'il y a au moins une extension a surveiller + */ + protected boolean isMonitored(Collection<String> exts) { + boolean result = false; + for (String ext : exts) { + result = extensions.contains(ext); + if (result) { + break; + } + } + return result; + } + + /** + * Add WikittyAccessStat in storage if necessary, one for each object + * + * @param securityToken + * @param wikitties + */ + protected void addStat(String securityToken, Collection wikitties) { + boolean userLoaded = false; + String user = null; + + List<Wikitty> stats = new ArrayList<Wikitty>(wikitties.size()); + for (Object o : wikitties) { + String id = null; + Collection<String> exts = null; + if (o instanceof Wikitty) { + Wikitty w = ((Wikitty)o); + id = w.getId(); + exts = w.getExtensionNames(); + } else if (o instanceof BusinessEntity) { + BusinessEntity e = ((BusinessEntity) o); + id = e.getWikittyId(); + exts = e.getExtensionNames(); + } + + if (exts != null && isMonitored(exts)) { + // on recupere le user que maintenant car potentiellement il n'y a + // rien a creer si aucun objet n'est dans la liste des extensions + // a surveiller + if (!userLoaded) { + user = getUserId(securityToken); + userLoaded = true; + } + + WikittyAccessStatImpl stat = new WikittyAccessStatImpl(); + stat.setToken(securityToken); + stat.setUser(user); + stat.setRestored(id); + + stats.add(stat.getWikitty()); + } + } + + getDelegate().store(securityToken, stats, false); + } + + @Override + public List<Wikitty> restore(String securityToken, List<String> id) { + List<Wikitty> result = super.restore(securityToken, id); + addStat(securityToken, result); + return result; + } + + @Override + public Map<WikittyTreeNode, Integer> restoreChildren(String securityToken, String wikittyId, Criteria filter) { + Map<WikittyTreeNode, Integer> result = + super.restoreChildren(securityToken, wikittyId, filter); + addStat(securityToken, result.keySet()); + return result; + } + + @Override + public Entry<WikittyTreeNode, Integer> restoreNode(String securityToken, String wikittyId, Criteria filter) { + Entry<WikittyTreeNode, Integer> result = + super.restoreNode(securityToken, wikittyId, filter); + addStat(securityToken, Collections.singleton(result.getKey())); + return result; + } + + @Override + public WikittyTree restoreTree(String securityToken, String wikittyId) { + WikittyTree result = super.restoreTree(securityToken, wikittyId); + + // before do some work, check if necessary + if (isMonitored(Collections.singleton(WikittyTreeNode.EXT_WIKITTYTREENODE))) { + List<WikittyTreeNode> nodes = new LinkedList<WikittyTreeNode>(); + List<WikittyTree> toVisit = new LinkedList<WikittyTree>(); + toVisit.add(result); + while (!toVisit.isEmpty()) { + WikittyTree tree = toVisit.get(0); + nodes.add(tree.getNode()); + toVisit.addAll(tree.getChildren()); + } + addStat(securityToken, nodes); + } + + return result; + } + + @Override + public Wikitty restoreVersion(String securityToken, String wikittyId, String version) { + Wikitty result = super.restoreVersion(securityToken, wikittyId, version); + addStat(securityToken, Collections.singleton(result)); + return result; + } + +} Modified: trunk/wikitty-api/src/main/xmi/wikitty.properties =================================================================== --- trunk/wikitty-api/src/main/xmi/wikitty.properties 2010-12-10 04:42:19 UTC (rev 590) +++ trunk/wikitty-api/src/main/xmi/wikitty.properties 2010-12-10 04:45:01 UTC (rev 591) @@ -29,3 +29,4 @@ org.nuiton.wikitty.entities.WikittyTreeNode.class.tagvalue.version=2.0 org.nuiton.wikitty.entities.WikittyUser.class.tagvalue.version=1.0 org.nuiton.wikitty.entities.WikittyI18n.class.tagvalue.version=1.0 +org.nuiton.wikitty.entities.WikittyAccessStat.class.tagvalue.version=1.0 Modified: trunk/wikitty-api/src/main/xmi/wikitty.zargo =================================================================== (Binary files differ)

1 0

r590 - trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 05:42:19 +0100 (Fri, 10 Dec 2010) New Revision: 590 Url: http://nuiton.org/repositories/revision/wikitty/590 Log: modif doc (table boder) Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java 2010-12-10 00:23:52 UTC (rev 589) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java 2010-12-10 04:42:19 UTC (rev 590) @@ -1,9 +1,12 @@ /** * <h1>Indexation</h1> * - * chaque type de champs est suffixe par un marqueur + * Ce module sert a l'indexation des wikitties dans SolR. Chaque champs d'un + * wikitty est indexe de differente facon selon son type. + * + * Chaque type de champs est suffixe par un marqueur. * - * <table> + * <table border="2"> * <tr> * <th>Type wikitty</th><th>Suffixe</th><th>Type d'indexation</th><th>valeur</th><th>stored</th><th>multiValued</th> * </tr> @@ -34,7 +37,7 @@ * </table> * * D'autres champs sont indexes - * <table> + * <table border="2"> * <tr> * <th>champs</th><th>Type d'indexation</th><th>valeur</th><th>stored</th><th>multiValued</th> * </tr>

1 0

r589 - trunk/wikitty-generators/src/main/java/org/nuiton/wikitty/generator
by bpoussin＠users.nuiton.org Dec. 10, 2010

Dec. 10, 2010

Author: bpoussin Date: 2010-12-10 01:23:52 +0100 (Fri, 10 Dec 2010) New Revision: 589 Url: http://nuiton.org/repositories/revision/wikitty/589 Log: Evolution #1137: create new field type binary (generation done) Modified: trunk/wikitty-generators/src/main/java/org/nuiton/wikitty/generator/WikittyTransformerUtil.java Modified: trunk/wikitty-generators/src/main/java/org/nuiton/wikitty/generator/WikittyTransformerUtil.java =================================================================== --- trunk/wikitty-generators/src/main/java/org/nuiton/wikitty/generator/WikittyTransformerUtil.java 2010-12-09 23:30:35 UTC (rev 588) +++ trunk/wikitty-generators/src/main/java/org/nuiton/wikitty/generator/WikittyTransformerUtil.java 2010-12-10 00:23:52 UTC (rev 589) @@ -137,7 +137,9 @@ asWhat = getCollectionTypeName(attribute); } else { String simpleTypeName = FQNtoSimpleName(attribute.getType()); - if (commonTypes.contains(simpleTypeName)) { + if (commonBinary.contains(simpleTypeName)) { + asWhat = "Bytes"; + } else if (commonTypes.contains(simpleTypeName)) { asWhat = StringUtils.capitalize(simpleTypeName); } else { asWhat = "Wikitty"; @@ -153,7 +155,9 @@ protected static String generateResultType(ObjectModelAttribute attribute, boolean considerMultiplicity) { String simpleTypeName = FQNtoSimpleName(attribute.getType()); - if (! commonTypes.contains(simpleTypeName)) { + if (commonBinary.contains(simpleTypeName)) { + simpleTypeName = "byte[]"; // return a wikitty Id + } else if (!commonTypes.contains(simpleTypeName)) { simpleTypeName = "String"; // return a wikitty Id } @@ -188,6 +192,8 @@ result = "Numeric"; } else if(commonStrings.contains(simpleType)) { result = "String"; + } else if(commonBinary.contains(simpleType)) { + result = "Binary"; } return result; } @@ -219,11 +225,22 @@ commonStrings.add(String.class.getSimpleName()); } + private static Set<String> commonBinary; + static { + commonBinary = new HashSet<String>(); + commonBinary.add("byte[]"); + // it's not a good idea to add this because this can clash with user + // object name, but in graphique modeler, we can't add byte[] as return + // type :( + commonBinary.add("Binary"); + } + private static Set<String> commonTypes; static { commonTypes = new HashSet<String>(); commonTypes.addAll(commonNumerics); commonTypes.addAll(commonStrings); + commonTypes.addAll(commonBinary); commonTypes.add(boolean.class.getSimpleName()); commonTypes.add(Boolean.class.getSimpleName()); commonTypes.add(Date.class.getSimpleName());

1 0

r588 - in trunk: . wikitty-api wikitty-api/src/main/java/org/nuiton/wikitty wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport wikitty-api/src/main/java/org/nuiton/wikitty/entities wikitty-api/src/main/java/org/nuiton/wikitty/search wikitty-api/src/main/java/org/nuiton/wikitty/services wikitty-api/src/test/java/org/nuiton/wikitty/conform wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc wikitty-jdbc-impl/src/main/resources wikitty-solr-impl/src/main/java/org/nuiton/wiki
by bpoussin＠users.nuiton.org Dec. 9, 2010

Dec. 9, 2010

Author: bpoussin Date: 2010-12-10 00:30:35 +0100 (Fri, 10 Dec 2010) New Revision: 588 Url: http://nuiton.org/repositories/revision/wikitty/588 Log: (un gros commit comme je ne les aimes pas :() Evolution #1137: create new field type binary (only generation is not done) Evolution #1141: Refactor solr code to store less field (currently 6 for 1 text field) Evolution #1142: Add documentation on solr indexation mecanisme Evolution #1143: Add mecanisme in jdbc to allow easy support of new database Change many string to constant value use BusinessHelper when we can to simplify code Added: trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query-mysql.properties trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/SolrUtil.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java trunk/wikitty-solr-impl/src/main/resources/solrconfig-complete.xml Modified: trunk/pom.xml trunk/wikitty-api/pom.xml trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyUtil.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportCSV.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportXML.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/FieldType.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/Search.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query.properties trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java trunk/wikitty-solr-impl/src/main/resources/schema.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/pom.xml 2010-12-09 23:30:35 UTC (rev 588) @@ -27,16 +27,25 @@ <module>wikitty-solr-impl</module> <module>wikitty-jdbc-impl</module> - <module>wikitty-ui-zk</module> <module>wikitty-hessian-client</module> <module>wikitty-hessian-server</module> <module>wikitty-perf-test</module> + <module>wikitty-publication</module> + </modules> <dependencyManagement> <dependencies> +  <dependency> + <groupId>net.iharder</groupId> + <artifactId>base64</artifactId> + <version>2.3.8</version> + </dependency> + +  + <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> <version>${jettyVersion}</version> @@ -57,42 +66,7 @@ <version>${jettyVersion}</version> </dependency> -  - - <dependency> <groupId>commons-dbcp</groupId> <artifactId>commons-dbcp</artifactId> <version>1.4</version> @@ -585,6 +559,7 @@ <module>wikitty-jpa-impl</module> <module>wikitty-jms-impl</module> <module>wikitty-multistorage-impl</module> + <module>wikitty-ui-zk</module> </modules> </profile> </profiles> Modified: trunk/wikitty-api/pom.xml =================================================================== --- trunk/wikitty-api/pom.xml 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/pom.xml 2010-12-09 23:30:35 UTC (rev 588) @@ -41,7 +41,13 @@ </dependency>  +  <dependency> + <groupId>net.iharder</groupId> + <artifactId>base64</artifactId> + </dependency> + + <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </dependency> Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyConfig.java 2010-12-09 23:30:35 UTC (rev 588) @@ -133,7 +133,9 @@ WIKITTY_STORAGE_JDBC_QUERY_FILE( "wikitty.storage.jdbc.queryfile", - _("JDBC query configuration file"), + _("JDBC query configuration file. You can put more than one file" + + " to load specific SQL statement for your database." + + " Exemple: wikitty-jdbc-query.properties,wikitty-jdbc-query-mysql.properties"), "wikitty-jdbc-query.properties", String.class, false, false), WIKITTY_STORAGE_JDBC_DRIVER( Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyUtil.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyUtil.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/WikittyUtil.java 2010-12-09 23:30:35 UTC (rev 588) @@ -25,8 +25,8 @@ package org.nuiton.wikitty; -import com.thoughtworks.xstream.converters.basic.DateConverter; import java.beans.PropertyDescriptor; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; import org.apache.commons.lang.StringUtils; @@ -60,6 +60,7 @@ import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; +import net.iharder.Base64; import org.apache.commons.beanutils.BeanUtilsBean; import org.apache.commons.beanutils.converters.DateTimeConverter; @@ -381,54 +382,37 @@ return result; } - // /** - // * - // * @param value null and empty string are casted to '0' int value. - // * @throws WikittyException on NumberFormatException or if value object - // can't be casted to int. - // */ - // static public int toInt(Object value) throws WikittyException { - // int result = 0; - // if (value == null || value.equals("") ) { - // result = 0; // default to 0 - // } else if (value instanceof Number) { - // result = ((Number) value).intValue(); - // } else { - // // try to convert to int - // try { - // result = Integer.parseInt(value.toString()); - // } catch (NumberFormatException eee) { - // throw new WikittyException(String.format( - // "Can't convert value '%s' to int", getClass(value)), eee); - // } - // } - // return result; - // } - // - // static public float toFloat(Object value) throws WikittyException { - // float result = 0; - // if (value == null) { - // result = 0; // default to 0 - // } else if (value instanceof Number) { - // result = ((Number) value).floatValue(); - // } else { - // // try to convert to float - // try { - // result = Float.parseFloat(value.toString()); - // } catch (NumberFormatException eee) { - // throw new WikittyException(String.format( - // "Can't convert value '%s' to float", getClass(value)), eee); - // } - // } - // return result; - // } + /** + * Get value as Binary. + * + * @param value null and empty return empty byte[] + * + * @return value as byte[] + */ + static public byte[] toBinary(Object value) { + byte[] result = null; + if (value == null || "".equals(value)) { + result = new byte[0]; // default to 0 + } else if (value instanceof String) { + try { + result = Base64.decode((String) value); + } catch (IOException eee) { + throw new WikittyException("Bad Base64 format", eee); + } + } else if (value instanceof byte[]) { + result = (byte[]) value; + } else { + result = value.toString().getBytes(); + } + return result; + } /** * Get value as BigDecimal. - * + * * @param value null and empty string are casted to '0' value. * @throws WikittyException on NumberFormatException or if value object can't be casted to number. - * + * * @return value as BigDecimal */ static public BigDecimal toBigDecimal(Object value) { @@ -565,29 +549,62 @@ } /** - * Convert object o for indexation. - * + * Convert object o for export CSV/XML. + * * @param field field description * @param o field value * @return solr representation */ - static public String toString(FieldType field, Object o) { + static public String toStringForExport(FieldType field, Object o) { String result = null; if (o != null) { + if (o instanceof String) { + result = (String) o; + } else { switch (field.getType()) { - case DATE: - // Date date = (Date)o; - result = (o instanceof String) ? (String) o - : WikittyUtil.solrDateFormat.format((Date) o); - break; - default: - result = WikittyUtil.toString(o); - break; + case BINARY: + result = Base64.encodeBytes((byte[]) o); + break; + case DATE: + // Date date = (Date)o; + result = WikittyUtil.solrDateFormat.format((Date) o); + break; + default: + result = WikittyUtil.toString(o); + break; + } } } return result; } + /** + * Convert object o for indexation + * + * @param field field description + * @param o field value + * @return solr representation + */ + static public String toStringForSearchEngine(FieldType field, Object o) { + String result = null; + if (o != null) { + switch (field.getType()) { + case BINARY: + // don't index binary + result = ""; + break; + case DATE: + // Date date = (Date)o; + result = (o instanceof String) ? (String) o + : WikittyUtil.solrDateFormat.format((Date) o); + break; + default: + result = WikittyUtil.toString(o); + break; + } + } + return result; + } /** * Convert string field representation to correct value type. @@ -599,18 +616,29 @@ static public Object fromString(FieldType field, String s) { Object result = null; switch (field.getType()) { - case BOOLEAN: - result = WikittyUtil.toBoolean(s); - break; - case DATE: - result = WikittyUtil.toDate(s); - break; - case NUMERIC: - result = WikittyUtil.toBigDecimal(s); - break; - default: - result = s; - break; + case BINARY: + if (s == null | "".equals(s)) { + result = new byte[0]; + } else { + try { + result = Base64.decode(s); + } catch (IOException eee) { + throw new WikittyException("Bad Base64 format", eee); + } + } + break; + case BOOLEAN: + result = WikittyUtil.toBoolean(s); + break; + case DATE: + result = WikittyUtil.toDate(s); + break; + case NUMERIC: + result = WikittyUtil.toBigDecimal(s); + break; + default: + result = s; + break; } return result; } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportCSV.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportCSV.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportCSV.java 2010-12-09 23:30:35 UTC (rev 588) @@ -344,13 +344,13 @@ if (fqField != null) { String separator = ""; for (Object o : (Collection<?>) fqField) { - String fqFieldValue = WikittyUtil.toString(type, o); + String fqFieldValue = WikittyUtil.toStringForExport(type, o); currentField += separator + "(" + fqFieldValue + ")"; separator = ","; } } } else { - String fqFieldValue = WikittyUtil.toString(type, w.getFqField(fieldName)); + String fqFieldValue = WikittyUtil.toStringForExport(type, w.getFqField(fieldName)); currentField = fqFieldValue; } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportXML.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportXML.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/addons/importexport/ImportExportXML.java 2010-12-09 23:30:35 UTC (rev 588) @@ -125,9 +125,8 @@ Map<String, String> tagValues = WikittyUtil.tagValuesToMap(CDATA); ext.setTagValues(tagValues); } else if (w != null) { - String[] fq = name.split("\\."); - String extensionName = fq[0]; - String fieldName = fq[1]; + String extensionName = WikittyExtension.extractExtensionName(name); + String fieldName = WikittyExtension.extractFieldName(name); FieldType fieldType = w.getFieldType(name); if (fieldType.isCollection()) { w.addToField(extensionName, fieldName, CDATA); @@ -177,7 +176,7 @@ Object fqField = w.getFqField(fieldName); if (fqField != null) { for (Object o : (Collection) fqField) { - String fqFieldValue = WikittyUtil.toString(type, o); + String fqFieldValue = WikittyUtil.toStringForExport(type, o); if (fqFieldValue != null) { fqFieldValue = StringEscapeUtils.escapeXml(fqFieldValue); result.write(" <" + fieldName + ">" + fqFieldValue + "</" + fieldName + ">\n"); @@ -185,7 +184,7 @@ } } } else { - String fqFieldValue = WikittyUtil.toString(type, w.getFqField(fieldName)); + String fqFieldValue = WikittyUtil.toStringForExport(type, w.getFqField(fieldName)); if (fqFieldValue != null) { fqFieldValue = StringEscapeUtils.escapeXml(fqFieldValue); result.write(" <" + fieldName + ">" + fqFieldValue + "</" + fieldName + ">\n"); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/FieldType.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/FieldType.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/FieldType.java 2010-12-09 23:30:35 UTC (rev 588) @@ -53,7 +53,7 @@ static public String NOT_NULL = "notNull"; static public enum TYPE { - BOOLEAN, DATE, NUMERIC, STRING, WIKITTY; + BINARY, BOOLEAN, DATE, NUMERIC, STRING, WIKITTY; /** * convert string to TYPE, this method accept not trimed and not well @@ -144,6 +144,8 @@ protected Object getContainedValidObject( Object value ) { Object result = null; switch (type) { + case BINARY: + result = WikittyUtil.toBinary(value); break; case DATE: result = WikittyUtil.toDate(value); break; case NUMERIC: Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/Wikitty.java 2010-12-09 23:30:35 UTC (rev 588) @@ -122,6 +122,8 @@ Object getFieldAsObject(String ext, String fieldName); + byte[] getFieldAsBytes(String ext, String fieldName); + boolean getFieldAsBoolean(String ext, String fieldName); BigDecimal getFieldAsBigDecimal(String ext, String fieldName); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyCopyOnWrite.java 2010-12-09 23:30:35 UTC (rev 588) @@ -234,6 +234,11 @@ } @Override + public byte[] getFieldAsBytes(String ext, String fieldName) { + return target.getFieldAsBytes(ext, fieldName); + } + + @Override public boolean getFieldAsBoolean(String ext, String fieldName) { return target.getFieldAsBoolean(ext, fieldName); } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/entities/WikittyImpl.java 2010-12-09 23:30:35 UTC (rev 588) @@ -501,6 +501,22 @@ } /* + * @see org.nuiton.wikitty.Wikitty#getFieldAsString(java.lang.String, java.lang.String) + */ + @Override + public byte[] getFieldAsBytes(String ext, String fieldName) { + Object value = getFieldAsObject(ext, fieldName); + try { + byte[] result = WikittyUtil.toBinary(value); + return result; + } catch (WikittyException eee) { + throw new WikittyException(String.format( + "field '%s' is not a valid byte[]", + ext + "." + fieldName), eee); + } + } + + /* * @see org.nuiton.wikitty.Wikitty#getFieldAsBoolean(java.lang.String, java.lang.String) */ @Override Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/Search.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/Search.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/search/Search.java 2010-12-09 23:30:35 UTC (rev 588) @@ -86,11 +86,11 @@ if (type.isCollection()) { Collection<?> collection = (Collection<?>) value; for (Object o : collection) { - String strValue = WikittyUtil.toString(type, o); + String strValue = WikittyUtil.toStringForSearchEngine(type, o); result.eq(fqfieldName, strValue); } } else { - String strValue = WikittyUtil.toString(type, value); + String strValue = WikittyUtil.toStringForSearchEngine(type, value); result.eq(fqfieldName, strValue); } } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/services/WikittyServiceStorage.java 2010-12-09 23:30:35 UTC (rev 588) @@ -161,7 +161,7 @@ if (type.isNotNull()) { if (null == w.getFieldAsObject(ext.getName(), fieldName)) { throw new WikittyException(String.format( - "Field %s must not be null", fieldName)); + "Field '%s' must not be null", fieldName)); } } } Modified: trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java =================================================================== --- trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-api/src/test/java/org/nuiton/wikitty/conform/StorageTest.java 2010-12-09 23:30:35 UTC (rev 588) @@ -61,6 +61,8 @@ import org.nuiton.wikitty.services.WikittyServiceEnhanced; import org.nuiton.wikitty.services.WikittyEvent; import org.nuiton.wikitty.WikittyUtil; +import org.nuiton.wikitty.entities.FieldFactory; +import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; import org.nuiton.wikitty.search.operators.Element; import org.nuiton.wikitty.search.Search; @@ -626,7 +628,8 @@ // store 1 Wikitty myWikitty = new WikittyImpl(); myWikitty.addExtension(WikittyTreeNodeImpl.extensions); - myWikitty.setField(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_NAME, "name"); + WikittyTreeNodeHelper.setName(myWikitty, "name"); + ws.store(null, Collections.singletonList(myWikitty), false); // delete @@ -641,4 +644,29 @@ WikittyServiceEnhanced.restore(ws, null, myWikitty.getId()); Assert.assertNotNull(restoredWikitty); } + + @Test + public void testBinaryFieldStorage() { + String extName = "BinaryExt"; + byte[] bytes = "Coucou le monde".getBytes(); + + WikittyExtension BinaryExt = new WikittyExtension(extName, + "1.0", // version + null, + WikittyUtil.buildFieldMapExtension( // building field map + "String name unique=\"true\"", + "Binary content")); + Wikitty w = new WikittyImpl(); + w.addExtension(BinaryExt); + w.setField(extName, "name", "LeBin"); + w.setField(extName, "content", bytes); + + ws.store(null, Collections.singletonList(w), false); + + Wikitty restoredWikitty = + WikittyServiceEnhanced.restore(ws, null, w.getId()); + Assert.assertNotNull(restoredWikitty); + Assert.assertEquals("LeBin", restoredWikitty.getFieldAsString(extName, "name")); + Assert.assertEquals(bytes, restoredWikitty.getFieldAsBytes(extName, "content")); + } } Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java =================================================================== --- trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyJDBCUtil.java 2010-12-09 23:30:35 UTC (rev 588) @@ -83,6 +83,8 @@ /** field type column in the extension_data table */ static final public String COL_FIELDTYPE = "fieldType"; /** boolean value column in the wikitty_data table */ + static final public String COL_BINARY_VALUE = "binaryValue"; + /** boolean value column in the wikitty_data table */ static final public String COL_BOOLEAN_VALUE = "booleanValue"; /** number value column in the wikitty_data table */ static final public String COL_NUMBER_VALUE = "numberValue"; @@ -109,9 +111,16 @@ "jdbc.queries.creation.wikitty.admin.test"; static final public String QUERY_CREATION_WIKITTY_ADMIN = "jdbc.queries.creation.wikitty.admin"; - /** wikitty_admin table creation query property name */ + /** wikitty_data column binary test exits query property name */ + static final public String QUERY_CREATION_WIKITTY_DATA_TEST_BINARY = + "jdbc.queries.creation.wikitty.data.test.binary"; + /** wikitty_data column binary creation with alter query property name */ + static final public String QUERY_CREATION_WIKITTY_DATA_ALTER_BINARY = + "jdbc.queries.creation.wikitty.data.alter.binary"; + /** wikitty_data table test exists query property name */ static final public String QUERY_CREATION_WIKITTY_DATA_TEST = "jdbc.queries.creation.wikitty.data.test"; + /** wikitty_data table creation query property name */ static final public String QUERY_CREATION_WIKITTY_DATA = "jdbc.queries.creation.wikitty.data"; /** insertion in the admin table query property name */ @@ -162,40 +171,49 @@ /** - * Loads the properties in the {@code wikitty-jdbc-config.properties} file. + * Loads the properties from configuration file, one or more properties + * can be load default load {@code wikitty-jdbc-config.properties} file. * * @param properties custom properties to override default configuration - * @return the properties for the connection and the queries + * @return the properties for the queries */ public static synchronized Properties loadQuery(ApplicationConfig config) { - Properties result = new Properties(); - - InputStream streamQuery = null; - try { + Properties result = null; - String wikittyQueryFile = config.getOption( - WikittyConfig.WikittyOption.WIKITTY_STORAGE_JDBC_QUERY_FILE.getKey()); - // queries - URL url = ClassLoader.getSystemResource(wikittyQueryFile); - if (url == null) { - ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); - url = contextClassLoader.getResource(wikittyQueryFile); + // list of all properties file to load + String wikittyQueryFileList = config.getOption( + WikittyConfig.WikittyOption.WIKITTY_STORAGE_JDBC_QUERY_FILE.getKey()); + + String[] wikittyQueryFiles = wikittyQueryFileList.split(","); + + for (String file : wikittyQueryFiles) { + // create new Properties with result as parent + result = new Properties(result); + InputStream streamQuery = null; + try { + // queries + URL url = ClassLoader.getSystemResource(file); + if (url == null) { + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + url = contextClassLoader.getResource(file); + } + + if (log.isInfoEnabled()) { + log.info("Reading resource from: " + url); + } + // url can't be null + streamQuery = url.openStream(); + result.load(streamQuery); + } catch (IOException eee) { + throw new WikittyException(String.format( + "Unable to load property file '%s'", + file), eee); + } finally { + IOUtils.closeQuietly(streamQuery); } - - if (log.isInfoEnabled()) { - log.info("Reading resource from: " + url); - } - // url can't be null - streamQuery = url.openStream(); - result.load(streamQuery); - - } catch (IOException eee) { - throw new WikittyException("Unable to load property file", eee); } - finally { - IOUtils.closeQuietly(streamQuery); - } + return result; } Modified: trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java =================================================================== --- trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-jdbc-impl/src/main/java/org/nuiton/wikitty/jdbc/WikittyStorageJDBC.java 2010-12-09 23:30:35 UTC (rev 588) @@ -24,6 +24,9 @@ */ package org.nuiton.wikitty.jdbc; +import java.io.IOException; +import java.io.InputStream; +import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.COL_BINARY_VALUE; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.COL_BOOLEAN_VALUE; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.COL_DATE_VALUE; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.COL_DELETION_DATE; @@ -36,6 +39,8 @@ import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CLEAR_WIKITTY; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CREATION_WIKITTY_ADMIN; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CREATION_WIKITTY_ADMIN_TEST; +import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CREATION_WIKITTY_DATA_ALTER_BINARY; +import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CREATION_WIKITTY_DATA_TEST_BINARY; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CREATION_WIKITTY_DATA; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_CREATION_WIKITTY_DATA_TEST; import static org.nuiton.wikitty.jdbc.WikittyJDBCUtil.QUERY_DELETE_WIKITTY_ADMIN; @@ -67,6 +72,7 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -109,8 +115,18 @@ public WikittyStorageJDBC(ApplicationConfig config, WikittyExtensionStorage extensionStorage) { this.config = config; this.extensionStorage = extensionStorage; - jdbcQuery = WikittyJDBCUtil.loadQuery(config); + checkTableOrCreation(); + + // all time use alter after creation for binaryValue column because + // this datatype is not portable + checkColumnBinaryOrAlter(); + } + + /** + * test table existance or create them if necessary + */ + protected void checkTableOrCreation() { Connection connectionTest = WikittyJDBCUtil.getJDBCConnection(config); try { // If test of existance work, no exception and do nothing @@ -139,9 +155,47 @@ } } + /** + * Add binary column if necessary + * If add can be done, wikitty work for all, except binary type + */ + protected void checkColumnBinaryOrAlter() { + Connection connectionTest = WikittyJDBCUtil.getJDBCConnection(config); + try { + // If test of existance work, no exception and do nothing + // if exception try to create databse + Statement statementTest = connectionTest.createStatement(); + statementTest.execute(jdbcQuery.getProperty(QUERY_CREATION_WIKITTY_DATA_TEST_BINARY)); + } catch (SQLException silentError) { + if (log.isInfoEnabled()) { + log.info("try to alter wikitty database to add binary column"); + } + Connection connection = WikittyJDBCUtil.getConnection(config); + try { + Statement statement = connection.createStatement(); + statement.execute(jdbcQuery.getProperty(QUERY_CREATION_WIKITTY_DATA_ALTER_BINARY)); + WikittyJDBCUtil.commitJDBCConnection(connection); + } catch (SQLException eee) { + WikittyJDBCUtil.rollbackJDBCConnection(connection); + // no exception just log fatal, wikitty can work without this + // column but can't store binary. If binary is not used there is + // no probleme + log.fatal("Can add column to store binary field. You can't use binary", eee); +// throw new WikittyException("Can't create table for wikitty storage", eee); + } finally { + WikittyJDBCUtil.closeQuietly(connection); + } + } finally { + WikittyJDBCUtil.closeQuietly(connectionTest); + } + } + protected String getColName(FieldType.TYPE type) { String result; switch(type) { + case BINARY: + result = COL_BINARY_VALUE; + break; case BOOLEAN: result = COL_BOOLEAN_VALUE; break; @@ -466,6 +520,14 @@ FieldType type = result.getFieldType(fqfieldName); Object value = null; switch (type.getType()) { + case BINARY: + InputStream blob = resultSet.getBinaryStream(COL_BINARY_VALUE); + try { + value = IOUtils.toByteArray(blob); + } catch (IOException eee) { + throw new WikittyException("Can't read blob stream for database", eee); + } + break; case BOOLEAN: value = resultSet.getBoolean(COL_BOOLEAN_VALUE); break; Added: trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query-mysql.properties =================================================================== --- trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query-mysql.properties (rev 0) +++ trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query-mysql.properties 2010-12-09 23:30:35 UTC (rev 588) @@ -0,0 +1,31 @@ +### +# #%L +# Wikitty :: wikitty-jdbc-impl +# +# $Id$ +# $HeadURL$ +# %% +# Copyright (C) 2010 CodeLutin, Benjamin Poussin +# %% +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Lesser Public License for more details. +# +# You should have received a copy of the GNU General Lesser Public +# License along with this program. If not, see +# <http://www.gnu.org/licenses/lgpl-3.0.html>. +# #L% +### +# +# ce fichier surcharge la requete d'ajout de la column binaryValue pour mysql +# il doit etre charge apres le fichier par defaut dans la sequence de fichier +# + +#table wikitty_data creation query +jdbc.queries.creation.wikitty.data.alter.binary=ALTER TABLE wikitty_data ADD COLUMN binaryValue blob; Modified: trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query.properties =================================================================== --- trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query.properties 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-jdbc-impl/src/main/resources/wikitty-jdbc-query.properties 2010-12-09 23:30:35 UTC (rev 588) @@ -22,6 +22,14 @@ # <http://www.gnu.org/licenses/lgpl-3.0.html>. # #L% ### +# +# Ce fichier essaie d'utiliser du sql portable d'une base \u00e0 une autre +# Il est compatible au moins avec les bases: +# - h2 (http://www.h2database.com) +# - PostgreSQL (http://www.postgresql.fr) +# - MySQL (http://www.mysql.com) excepter l'ajout de la colonne binaryValue (a cause du bytea) +# + #table extension_admin creation query jdbc.queries.creation.extension.admin.test=SELECT * FROM extension_admin LIMIT 1; jdbc.queries.creation.extension.admin=CREATE TABLE extension_admin (\ @@ -51,6 +59,9 @@ PRIMARY KEY (id)); #table wikitty_data creation query +# all time use alter after creation for binaryValue column because this datatype is not portable +jdbc.queries.creation.wikitty.data.test.binary=SELECT binaryValue FROM wikitty_data LIMIT 1; +jdbc.queries.creation.wikitty.data.alter.binary=ALTER TABLE wikitty_data ADD COLUMN binaryValue bytea; jdbc.queries.creation.wikitty.data.test=SELECT * FROM wikitty_data LIMIT 1; jdbc.queries.creation.wikitty.data=CREATE TABLE wikitty_data (\ id varchar(64) NOT NULL,\ Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/Restriction2Solr.java 2010-12-09 23:30:35 UTC (rev 588) @@ -276,13 +276,13 @@ private String like2solr(Like like) throws WikittyException { SearchAs searchAs = like.getSearchAs(); String element2solr = element2solr(like.getElement()); - if(element2solr.endsWith("_s")) { // is string + if(element2solr.endsWith(WikittySearchEngineSolr.SUFFIX_STRING)) { // is string switch(searchAs) { case AsText: - element2solr += "_t"; + element2solr += WikittySearchEngineSolr.SUFFIX_STRING_FULLTEXT; break; case ToLowerCase: - element2solr += "_c"; + element2solr += WikittySearchEngineSolr.SUFFIX_STRING_LOWERCASE; break; } } @@ -290,7 +290,7 @@ // Warning if you need add searchAs, AsText and ToLowerCase need search // at lowercase String value2solr = value2solr(like.getValue()); - if(!element2solr.endsWith("_dt")) { // is not date + if(!element2solr.endsWith(WikittySearchEngineSolr.SUFFIX_DATE)) { // is not date value2solr = value2solr.toLowerCase(); } @@ -300,13 +300,13 @@ private String unlike2solr(Unlike unlike) throws WikittyException { SearchAs searchAs = unlike.getSearchAs(); String element2solr = element2solr(unlike.getElement()); - if(element2solr.endsWith("_s")) { // is string + if(element2solr.endsWith(WikittySearchEngineSolr.SUFFIX_STRING)) { // is string switch(searchAs) { case AsText: - element2solr += "_t"; + element2solr += WikittySearchEngineSolr.SUFFIX_STRING_FULLTEXT; break; case ToLowerCase: - element2solr += "_c"; + element2solr += WikittySearchEngineSolr.SUFFIX_STRING_LOWERCASE; break; } } @@ -314,7 +314,7 @@ // Warning if you need add searchAs, AsText and ToLowerCase need search // at lowercase String value2solr = value2solr(unlike.getValue()); - if(!element2solr.endsWith("_dt")) { // is not date + if(!element2solr.endsWith(WikittySearchEngineSolr.SUFFIX_DATE)) { // is not date value2solr = value2solr.toLowerCase(); } Added: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/SolrUtil.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/SolrUtil.java (rev 0) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/SolrUtil.java 2010-12-09 23:30:35 UTC (rev 588) @@ -0,0 +1,78 @@ +package org.nuiton.wikitty.solr; + + +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class SolrUtil { + + /** to use log facility, just put in your code: log.info(\"...\"); */ + static private Log log = LogFactory.getLog(SolrUtil.class); + + /** + * copy all field of source in new document. + * If include is true copy only field specified in fields + * if include is false copy all field except field in fields. + * + * example: + * if doc contains field: abc, aabbcc, aaabbbccc, toto + * copySolrDocument(doc, true, "aa.*", ".*bbb.*") + * field copied are: aabbcc, aaabbbccc + * + * copySolrDocument(doc, false, "aa.*", ".*bbb.*") + * field copied are: abc, toto + * + * @param source + * @param include + * @param fields + * @return + */ + static public SolrInputDocument copySolrDocument( + SolrDocument source, boolean include, String... fields) { + SolrInputDocument result = new SolrInputDocument(); + Collection<String> fieldNames = source.getFieldNames(); + + Set<String> fieldToCopy = new HashSet<String>(); + if (include) { + for (String fieldName : fieldNames) { + for (String fieldRegexp : fields) { + if (fieldName.matches(fieldRegexp)) { + fieldToCopy.add(fieldName); + } + } + } + } else { // exclude + fieldToCopy.addAll(fieldNames); + for (String fieldName : fieldNames) { + for (String fieldRegexp : fields) { + if (fieldName.matches(fieldRegexp)) { + fieldToCopy.remove(fieldName); + } + } + } + } + + for (String fieldName : fieldToCopy) { + Collection<Object> fieldValues = source.getFieldValues(fieldName); + for (Object fieldValue : fieldValues) { + result.addField(fieldName, fieldValue); + } + } + return result; + } +} Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/WikittySearchEngineSolr.java 2010-12-09 23:30:35 UTC (rev 588) @@ -69,8 +69,10 @@ import com.arjuna.ats.arjuna.state.OutputObjectState; import com.arjuna.ats.internal.arjuna.abstractrecords.LastResourceRecord; import java.io.File; +import java.util.regex.Pattern; import org.nuiton.util.ApplicationConfig; import org.nuiton.wikitty.WikittyConfig; +import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; /** @@ -107,6 +109,31 @@ static final public String TREENODE_ROOT = TREENODE_PREFIX + "root"; static final public String TREENODE_PATH = TREENODE_PREFIX + "path"; + static final public String SUFFIX_BINARY = "_bi"; + static final public String SUFFIX_BOOLEAN = "_b"; + static final public String SUFFIX_DATE = "_dt"; + static final public String SUFFIX_STRING = "_s"; + static final public String SUFFIX_NUMERIC = "_d"; + static final public String SUFFIX_WIKITTY = "_w"; // not used yet + static final public String SUFFIX_STRING_LOWERCASE = "_c"; + static final public String SUFFIX_STRING_FULLTEXT = "_t"; + + static final public String[] fieldNotToCopyPattern = { + Pattern.quote(TREENODE_PREFIX) + ".*" + }; + + static final public String[] fieldToCopyPattern = { + // match: id, extensions, not_null_fields + SOLR_ID, SOLR_EXTENSIONS, SOLR_NOT_NULL_FIELDS, + // match: "(?!(all\.)).*_bi" accept ce qui fini par _bi sauf si ca commence par "all." + "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_BINARY, + "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_BOOLEAN, + "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_DATE, + "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_NUMERIC, + "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_STRING, + "(?!(" + SOLR_ALL_EXTENSIONS + "\\.)).*" + SUFFIX_WIKITTY + }; + /** use to permit client to modify fieldname during query generation */ static public interface FieldModifier { public String convertToSolr(WikittyTransaction transaction, String fieldname); @@ -122,7 +149,7 @@ @Override public String convertToSolr(WikittyTransaction transaction, String fqfieldname) { String result = fqfieldname; - String[] searchField = fqfieldname.split("\\."); + String[] searchField = fqfieldname.split(WikittyUtil.FQ_FIELD_NAME_SEPARATOR_REGEX); if (Element.ELT_EXTENSION.equals(fqfieldname)) { result = SOLR_EXTENSIONS; @@ -135,10 +162,12 @@ String fieldName = searchField[1]; if (Criteria.ALL_EXTENSIONS.equals(extName)) { - fqfieldname = SOLR_ALL_EXTENSIONS + "." + fieldName; + fqfieldname = SOLR_ALL_EXTENSIONS + + WikittyUtil.FQ_FIELD_NAME_SEPARATOR + fieldName; } if (searchField.length >= 3) { + // TODO poussin 20101209 je ne vois pas dans quel cas on passe ici String fieldNameType = searchField[2]; TYPE type = FieldType.TYPE.valueOf(fieldNameType); result = WikittySearchEngineSolr.getSolrFieldName(fqfieldname, type); @@ -162,10 +191,21 @@ return result; } + /** + * if you change this, change + * {@link WikittySearchEngineSolr#getSolrFieldName(java.lang.String, org.nuiton.wikitty.entities.FieldType.TYPE)} + * too + */ @Override public String convertToField(WikittyTransaction transaction, String solrName) { - String fieldName = solrName.replaceAll("(_b$)|(_dt$)|(_s$)|(_d$)", ""); - if(SOLR_EXTENSIONS.equals(fieldName)) { + String fieldName = solrName.replaceAll( + "(" + SUFFIX_BINARY + "$)" + + "|(" + SUFFIX_BOOLEAN + "$)" + + "|(" + SUFFIX_DATE + "$)" + + "|(" + SUFFIX_STRING + "$)" + + "|(" + SUFFIX_WIKITTY + "$)" + + "|(" + SUFFIX_NUMERIC + "$)", ""); + if (SOLR_EXTENSIONS.equals(fieldName)) { fieldName = Element.ELT_EXTENSION; } return fieldName; @@ -292,22 +332,11 @@ // Get documents SolrInputDocument doc = solrResource.getAddedDoc(id); if(doc == null) { - doc = new SolrInputDocument(); - // Copy old field value SolrDocument found = findById(solrServer, id); if (found != null) { - Collection<String> fieldNames = found.getFieldNames(); - for (String fieldName : fieldNames) { - Collection<Object> fieldValues = found.getFieldValues(fieldName); - - if(!fieldName.startsWith(TREENODE_PREFIX)) { - for (Object fieldValue : fieldValues) { - doc.addField(fieldName, fieldValue); - } - } - } - + // FIXME poussin 20101209 a finir en ne copiant que le necessaire voir javadoc du package + doc = SolrUtil.copySolrDocument(found, false, fieldNotToCopyPattern); solrResource.addDoc(id, doc); } else { if (log.isWarnEnabled()) { @@ -639,7 +668,8 @@ query.setStart(firstIndex); int nbRows; if (endIndex == -1) { - // WARNING It is necessary to substract 'start' otherwise, there is a capacity overlow in solR + // WARNING It is necessary to substract 'start' otherwise, + // there is a capacity overlow in solR nbRows = Integer.MAX_VALUE - firstIndex; } else { nbRows = endIndex - firstIndex + 1; @@ -752,7 +782,7 @@ public Integer findNodeCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { String wikittyId = w.getId(); - String parent = w.getFieldAsWikitty(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); + String parent = WikittyTreeNodeHelper.getParent(w); if(parent == null) { parent = TREENODE_ROOT; } else { @@ -773,7 +803,7 @@ public Map<String, Integer> findAllChildrenCount(WikittyTransaction transaction, Wikitty w, Criteria filter) { String wikittyId = w.getId(); - String parent = w.getFieldAsWikitty(WikittyTreeNode.EXT_WIKITTYTREENODE, WikittyTreeNode.FIELD_WIKITTYTREENODE_PARENT); + String parent = WikittyTreeNodeHelper.getParent(w); if(parent == null) { parent = TREENODE_ROOT; } else { @@ -823,11 +853,39 @@ * extensions : extensionNames * fieldName : fieldValue * + * FIXME poussin 20101209 beaucoup trop de champs redondant sauver. + * + * Une champs de type chaine est indexer 7 fois :(. _s et _s_c sont indexe + * exactement de la meme facon sauf qu'avant l'envoi a solr _s_c en mis en + * minuscule (ce qui devrait etre le role de solr en mettant la bonne config) + * _s et _s_c sont indexer en string et _s_t est indexe en text. + * + * string est indexer sans traitement + * text est indexer apres traitement + * (ce qui permet lors de la recherche des ecarts. ex: wifi matchera wi-fi) + * + * _s_c et _s_t ne sont utilise que pour + * {@link Restriction2Solr#like2solr}{@link Restriction2Solr#unlike2solr} + * n'y a-t-il pas moyen d'utiliser un champs deja indexe ? + * + * On a aujourd'hui: + * <li> text (qui rassemble tous les champs et est le champs de recherche par defaut) + * <li> ext.field_s (verbatime) + * <li> ext.field_s_c (verbatime lowercase) + * <li> ext.field_s_t (travaille) + * <li> all.field_s (verbatime) + * <li> all.field_s_c (verbatime lowercase) + * <li> all.field_s_t (travaille) + * + * et surtout ils sont tous stored :( + * * @param w all wikitties object to index * @return solrInputDocument used to modify index */ protected SolrInputDocument createIndexDocument(Wikitty w) { - log.debug("index wikitty " + w.getId()); + if (log.isDebugEnabled()) { + log.debug("index wikitty " + w.getId()); + } SolrInputDocument doc = new SolrInputDocument(); String id = w.getId(); @@ -842,8 +900,9 @@ TYPE type = fieldType.getType(); String solrFqFieldName = getSolrFieldName(fqfieldName, type); - String[] solrFieldName = solrFqFieldName.split("\\."); - String solrAllFieldName = SOLR_ALL_EXTENSIONS + "." + solrFieldName[1]; + String solrAllFieldName = SOLR_ALL_EXTENSIONS + + WikittyUtil.FQ_FIELD_NAME_SEPARATOR + + WikittyUtil.getFieldNameFromFQFieldName(solrFqFieldName); Object objectValue = w.getFqField(fqfieldName); if(objectValue != null) { @@ -856,11 +915,11 @@ // Store string field in differents styles if(type == TYPE.STRING) { - doc.addField(solrFqFieldName + "_t", itemValue); - doc.addField(solrAllFieldName + "_t", itemValue); + doc.addField(solrFqFieldName + SUFFIX_STRING_FULLTEXT, itemValue); + doc.addField(solrAllFieldName + SUFFIX_STRING_FULLTEXT, itemValue); String itemValueLowerCase = itemValue.toString().toLowerCase(); - doc.addField(solrFqFieldName + "_c", itemValueLowerCase); - doc.addField(solrAllFieldName + "_c", itemValueLowerCase); + doc.addField(solrFqFieldName + SUFFIX_STRING_LOWERCASE, itemValueLowerCase); + doc.addField(solrAllFieldName + SUFFIX_STRING_LOWERCASE, itemValueLowerCase); } doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); @@ -873,15 +932,17 @@ // Store string field in differents styles if(type == TYPE.STRING) { - doc.addField(solrFqFieldName + "_t", objectValue); - doc.addField(solrAllFieldName + "_t", objectValue); + doc.addField(solrFqFieldName + SUFFIX_STRING_FULLTEXT, objectValue); + doc.addField(solrAllFieldName + SUFFIX_STRING_FULLTEXT, objectValue); String objectValueLowerCase = objectValue.toString().toLowerCase(); - doc.addField(solrFqFieldName + "_c", objectValueLowerCase); - doc.addField(solrAllFieldName + "_c", objectValueLowerCase); + doc.addField(solrFqFieldName + SUFFIX_STRING_LOWERCASE, objectValueLowerCase); + doc.addField(solrAllFieldName + SUFFIX_STRING_LOWERCASE, objectValueLowerCase); } doc.addField(SOLR_NOT_NULL_FIELDS, fqfieldName); - log.debug("index field " + solrFqFieldName + " with value '" + objectValue + "'"); + if (log.isDebugEnabled()) { + log.debug("index field " + solrFqFieldName + " with value '" + objectValue + "'"); + } } } } @@ -909,16 +970,28 @@ return null; } + /** + * if you change this method, change + * {@link TypeFieldModifer#convertToField(org.nuiton.wikitty.services.WikittyTransaction, java.lang.String)} + * too + * + * @param fqfieldName + * @param type + * @return + */ public static String getSolrFieldName(String fqfieldName, TYPE type) { switch (type) { + case BINARY: + return fqfieldName + SUFFIX_BINARY; case BOOLEAN: - return fqfieldName + "_b"; + return fqfieldName + SUFFIX_BOOLEAN; case DATE: - return fqfieldName + "_dt"; + return fqfieldName + SUFFIX_DATE; case STRING: - return fqfieldName + "_s"; + return fqfieldName + SUFFIX_STRING; case NUMERIC: - return fqfieldName + "_d"; + return fqfieldName + SUFFIX_NUMERIC; + // FIXME poussin 20101209 pourquoi ne pas mettre explicitement un suffix pour le type WIKITTY ? default: return fqfieldName; } Added: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java (rev 0) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/solr/package-info.java 2010-12-09 23:30:35 UTC (rev 588) @@ -0,0 +1,130 @@ +/** + * <h1>Indexation</h1> + * + * chaque type de champs est suffixe par un marqueur + * + * <table> + * <tr> + * <th>Type wikitty</th><th>Suffixe</th><th>Type d'indexation</th><th>valeur</th><th>stored</th><th>multiValued</th> + * </tr> + * <tr> + * <td>{@link org.nuiton.wikitty.entities.FieldType.TYPE#BINARY}</td><td>_bi {@link WikittySearchEngineSolr#SUFFIX_BINARY}</td><td>aucun</td><td>vide</td><td>non</td><td>true</td> + * </tr> + * <tr> + * <td>{@link org.nuiton.wikitty.entities.FieldType.TYPE#BOOLEAN}</td><td>_b {@link WikittySearchEngineSolr#SUFFIX_BOOLEAN}</td><td>boolean</td><td>la valeur du champs</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>{@link org.nuiton.wikitty.entities.FieldType.TYPE#DATE}</td><td>_dt {@link WikittySearchEngineSolr#SUFFIX_DATE}</td><td>date</td><td>la valeur du champs</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>{@link org.nuiton.wikitty.entities.FieldType.TYPE#NUMBER}</td><td>_d {@link WikittySearchEngineSolr#SUFFIX_NUMERIC}</td><td>sdouble</td><td>la valeur du champs</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>{@link org.nuiton.wikitty.entities.FieldType.TYPE#WIKITTY}</td><td>_w {@link WikittySearchEngineSolr#SUFFIX_WIKITTY}</td><td>string</td><td>l'id du wikitty</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td rowspan="3">{@link org.nuiton.wikitty.entities.FieldType.TYPE#STRING}</td><td>_s {@link WikittySearchEngineSolr#SUFFIX_STRING}</td><td>string</td><td>la valeur du champs</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>_s_c {@link WikittySearchEngineSolr#SUFFIX_STRING_LOWERCASE}</td><td>string</td><td>la valeur du champs en minuscule</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>_s_t {@link WikittySearchEngineSolr#SUFFIX_STRING_FULLTEXT}</td><td>text</td><td>la valeur du champs</td><td>true</td><td>true</td> + * </tr> + * </table> + * + * D'autres champs sont indexes + * <table> + * <tr> + * <th>champs</th><th>Type d'indexation</th><th>valeur</th><th>stored</th><th>multiValued</th> + * </tr> + * <tr> + * <td>id</td><td>string</td><td>l'id du wikitty</td><td>true</td><td>false</td> + * </tr> + * <tr> + * <td>extensions</td><td>string</td><td>la liste des extensions</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>not_null_fields</td><td>string</td><td>la liste des champs qui doivent etre non null</td><td>true</td><td>true</td> + * </tr> + * <tr> + * <td>text</td><td>text</td><td>la valeur de tous les champs ayant un suffix</td><td>true</td><td>true</td> + * </tr> + * </table> + * + * <p> + * Les champs sont tous restockes dans une extension 'all' pour pouvoir faire + * des recherches sur toutes les extensions en meme temps. Par exemple rechercher + * tout ce qui porte le 'nom' 'portable' quelque soit l'extension (*.nom:portable) + * <p> + * Les chaines de caracteres doivent obligatoirement etre indexee en type string + * si l'on veut pouvoir faire des facettes dessus. Il faut donc obligatoirement + * indexer les chaines en 'string' et aussi en 'text' pour pouvoir les utiliser + * dans les facettes mais aussi que la recheche soit plus permissive. + * <p> + * SolR copie tous les champs dans le champs 'text' pour la recherche fulltext + * ce champs est le champs par defaut de recherche. + * <p> + * id est marque comme devant etre un champs unique (et donc lorsqu'on enregistre + * un nouveau document avec le meme id, l'ancien est supprime) + * <p> + * Tous les champs sont marque stored car lors de la reindexation des arbres + * on a besoin de faire une copie de l'ancien document et donc de pouvoir + * recuperer la valeur des champs (voir alternative) + * <p> + * Par exemple si on a un champs <b>product.description: String</b> nous le + * retrouverons dans 7 champs de l'index: + * + * <li> text : text (car est la copie de tous les champs) + * <li> product.description_s : string (necessaire pour la facetisation) + * <li> product.description_s_c : string + * <li> product.description_s_t : text + * <li> all.description_s : string + * <li> all.description_s_c : string + * <li> all.description_s_t : text + * + * il faudrait que les 5 derniers soit autogenere par solr en utilisant un + * <b>copyField</b> dans le schema.xml et qu'il ne soit pas stocke. Mais pour + * cela il faudrait que <b>copyField</b> permette l'utilisation de regexp + * (faire un patch a SolR ?) + * + * <copyField source="*_s" dest="*_s_c"/> + * <copyField source="*_s" dest="*_s_t"/> + * + * <copyField source="*.*_s" dest="all.*_s"/> + * <copyField source="*.*_s" dest="all.*_s_c"/> + * <copyField source="*.*_s" dest="all.*_s_t"/> + * + * et aussi definir les all pour les autres types + * <copyField source="*.*_b" dest="all.*_b"/> + * <copyField source="*.*_dt" dest="all.*_dt"/> + * <copyField source="*.*_d" dest="all.*_d"/> + * <copyField source="*.*_w" dest="all.*_w"/> + * + * copyField ne support que une * et au debut ou a la fin, donc actuellement + * il serait possible de d'avoir + * + * <copyField source="*_s" dest="*_s_c"/> + * <copyField source="*_s" dest="*_s_t"/> + * + * et d'enregistrer les champs deux fois en <b>extName.fieldName</b> et en + * <b>all.fieldName</b> ensuite lorsqu'on a besoin de copier un document + * il ne faut prendre les champs id, extensions, not_null_fields et les champs + * ne commencant pas par 'all.' et se finissant exclusivement par + * _bi, _b, _dt, _d, _s, _w. Cest dernier seront enregistrer aussi en all. + * + * Ainsi on passe de 6 champs stocke + 1, a 1 champ stocker + 6. Pour les chaines + * et de 2 champs stockes a 1 champ stocke + 1. + * + * + * <h2>alternative au stockage de tout les champs</h2> + * <p> + * Une alternative serait de ne reprendre que les champs reels (pas les copies) + * et recreer les copies a partir de ceux la. Les copies pourront ne plus etre + * stored=true. + * <p> + * Une autre alternative serait de récuperer l'objet dans le Storage et de le + * reindexer completement + * + */ +package org.nuiton.wikitty.solr; Modified: trunk/wikitty-solr-impl/src/main/resources/schema.xml =================================================================== --- trunk/wikitty-solr-impl/src/main/resources/schema.xml 2010-12-09 15:14:22 UTC (rev 587) +++ trunk/wikitty-solr-impl/src/main/resources/schema.xml 2010-12-09 23:30:35 UTC (rev 588) @@ -289,10 +289,12 @@ <copyField source="*_d" dest="text"/> <copyField source="*_dt" dest="text"/> +  + <dynamicField name="*_bi" type="string" indexed="false" stored="false" multiValued="true"/> <dynamicField name="*_i" type="sint" indexed="true" stored="true" multiValued="true"/> <dynamicField name="*_s" type="string" indexed="true" stored="true" multiValued="true"/> - <dynamicField name="*_t" type="text" indexed="true" stored="true" multiValued="true"/> - <dynamicField name="*_c" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_t" type="text" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_c" type="string" indexed="true" stored="true" multiValued="true"/> <dynamicField name="*_l" type="slong" indexed="true" stored="true" multiValued="true"/> <dynamicField name="*_b" type="boolean" indexed="true" stored="true" multiValued="true"/> <dynamicField name="*_f" type="sfloat" indexed="true" stored="true" multiValued="true"/> Added: trunk/wikitty-solr-impl/src/main/resources/solrconfig-complete.xml =================================================================== --- trunk/wikitty-solr-impl/src/main/resources/solrconfig-complete.xml (rev 0) +++ trunk/wikitty-solr-impl/src/main/resources/solrconfig-complete.xml 2010-12-09 23:30:35 UTC (rev 588) @@ -0,0 +1,733 @@ +<?xml version="1.0" encoding="UTF-8" ?> + + + + +<config> +  + <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError> + +  + <dataDir>${wikitty.searchengine.solr.directory.data:./solr/data}</dataDir> + +  + <directoryFactory name="DirectoryFactory" class="${wikitty.searchengine.solr.directory.factory:solr.StandardDirectoryFactory}"/> + + <indexDefaults> +  + <useCompoundFile>false</useCompoundFile> + + <mergeFactor>10</mergeFactor> +  +  +  + <ramBufferSizeMB>32</ramBufferSizeMB> + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + <writeLockTimeout>1000</writeLockTimeout> + <commitLockTimeout>10000</commitLockTimeout> + +  +  +  +  + +  +  + +  + <lockType>simple</lockType> + </indexDefaults> + + <mainIndex> +  + <useCompoundFile>false</useCompoundFile> + <ramBufferSizeMB>32</ramBufferSizeMB> + <mergeFactor>10</mergeFactor> +  +  + <maxMergeDocs>2147483647</maxMergeDocs> + <maxFieldLength>10000</maxFieldLength> + +  + <unlockOnStartup>true</unlockOnStartup> + </mainIndex> + +  + <jmx /> + +  + <updateHandler class="solr.DirectUpdateHandler2"> + +  + +  + +  +  +  + + </updateHandler> + + + <query> +  + <maxBooleanClauses>1024</maxBooleanClauses> + + +  + <filterCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="128"/> + +  + <queryResultCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="32"/> + +  + <documentCache + class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + +  + <enableLazyFieldLoading>true</enableLazyFieldLoading> + +  +  + +  + +  + <queryResultWindowSize>50</queryResultWindowSize> + +  + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + +  + <HashDocSet maxSize="3000" loadFactor="0.75"/> + +  +  + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + +  + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst> + <lst><str name="q">static firstSearcher warming query from solrconfig.xml</str></lst> + </arr> + </listener> + +  + <useColdSearcher>false</useColdSearcher> + +  + <maxWarmingSearchers>2</maxWarmingSearchers> + + </query> + +  + <requestDispatcher handleSelect="true" > +  + <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" /> + +  +  + <httpCaching lastModifiedFrom="openTime" + etagSeed="Solr"> +  +  +  + </httpCaching> + </requestDispatcher> + + +  + <requestHandler name="standard" class="solr.SearchHandler" default="true"> +  + <lst name="defaults"> + <str name="echoParams">explicit</str> +  + </lst> + </requestHandler> + + +  + <requestHandler name="dismax" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <float name="tie">0.01</float> + <str name="qf"> + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + </str> + <str name="pf"> + text^0.2 features^1.1 name^1.5 manu^1.4 manu_exact^1.9 + </str> + <str name="bf"> + ord(popularity)^0.5 recip(rord(price),1,1000,1000)^0.3 + </str> + <str name="fl"> + id,name,price,score + </str> + <str name="mm"> + 2<-1 5<-2 6<90% + </str> + <int name="ps">100</int> + <str name="q.alt">*:*</str> +  + <str name="hl.fl">text features name</str> +  + <str name="f.name.hl.fragsize">0</str> +  + <str name="f.name.hl.alternateField">name</str> + <str name="f.text.hl.fragmenter">regex</str>  + </lst> + </requestHandler> + +  + <requestHandler name="partitioned" class="solr.SearchHandler" > + <lst name="defaults"> + <str name="defType">dismax</str> + <str name="echoParams">explicit</str> + <str name="qf">text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0</str> + <str name="mm">2<-1 5<-2 6<90%</str> +  + <str name="bq">incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2</str> + </lst> +  + <lst name="appends"> + <str name="fq">inStock:true</str> + </lst> +  + <lst name="invariants"> + <str name="facet.field">cat</str> + <str name="facet.field">manu_exact</str> + <str name="facet.query">price:[* TO 500]</str> + <str name="facet.query">price:[500 TO *]</str> + </lst> + </requestHandler> + + +  + +  + <searchComponent name="spellcheck" class="solr.SpellCheckComponent"> + + <str name="queryAnalyzerFieldType">textSpell</str> + + <lst name="spellchecker"> + <str name="name">default</str> + <str name="field">spell</str> + <str name="spellcheckIndexDir">./spellchecker1</str> + + </lst> + <lst name="spellchecker"> + <str name="name">jarowinkler</str> + <str name="field">spell</str> +  + <str name="distanceMeasure">org.apache.lucene.search.spell.JaroWinklerDistance</str> + <str name="spellcheckIndexDir">./spellchecker2</str> + + </lst> + + <lst name="spellchecker"> + <str name="classname">solr.FileBasedSpellChecker</str> + <str name="name">file</str> + <str name="sourceLocation">spellings.txt</str> + <str name="characterEncoding">UTF-8</str> + <str name="spellcheckIndexDir">./spellcheckerFile</str> + </lst> + </searchComponent> + +  + <requestHandler name="/spellCheckCompRH" class="solr.SearchHandler"> + <lst name="defaults"> +  + <str name="spellcheck.onlyMorePopular">false</str> +  + <str name="spellcheck.extendedResults">false</str> +  + <str name="spellcheck.count">1</str> + </lst> + <arr name="last-components"> + <str>spellcheck</str> + </arr> + </requestHandler> + +  + + +  + + +  + <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" /> + +  + <requestHandler name="/analysis" class="solr.AnalysisRequestHandler" /> + + +  + <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" /> + + +  + <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" /> + +  + <requestHandler name="/admin/ping" class="PingRequestHandler"> + <lst name="defaults"> + <str name="qt">standard</str> + <str name="q">solrpingquery</str> + <str name="echoParams">all</str> + </lst> + </requestHandler> + +  + <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" > + <lst name="defaults"> + <str name="echoParams">explicit</str>  + <str name="echoHandler">true</str> + </lst> + </requestHandler> + + <highlighting> +  +  + <fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true"> + <lst name="defaults"> + <int name="hl.fragsize">100</int> + </lst> + </fragmenter> + +  + <fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter"> + <lst name="defaults"> +  + <int name="hl.fragsize">70</int> +  + <float name="hl.regex.slop">0.5</float> +  + <str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str> + </lst> + </fragmenter> + +  + <formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true"> + <lst name="defaults"> + <str name="hl.simple.pre"><![CDATA[<em>]]></str> + <str name="hl.simple.post"><![CDATA[</em>]]></str> + </lst> + </formatter> + </highlighting> + + +  + +  + <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter"> + <int name="xsltCacheLifetimeSeconds">5</int> + </queryResponseWriter> + + + <queryParser name="wikitty" class="org.nuiton.wikitty.solr.WikittyQueryParser"/> + +  + +  + +  + <admin> + <defaultQuery>solr</defaultQuery> + +  + </admin> + +</config>

1 0

r587 - in trunk: . wikitty-perf-test wikitty-perf-test/src wikitty-perf-test/src/main wikitty-perf-test/src/main/java wikitty-perf-test/src/main/java/org wikitty-perf-test/src/main/java/org/nuiton wikitty-perf-test/src/main/java/org/nuiton/wikitty wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest
by vbriand＠users.nuiton.org Dec. 9, 2010

Dec. 9, 2010

Author: vbriand Date: 2010-12-09 16:14:22 +0100 (Thu, 09 Dec 2010) New Revision: 587 Url: http://nuiton.org/repositories/revision/wikitty/587 Log: Added wikitty-perf-test module Added: trunk/wikitty-perf-test/ trunk/wikitty-perf-test/LICENSE.txt trunk/wikitty-perf-test/README.txt trunk/wikitty-perf-test/changelog.txt trunk/wikitty-perf-test/pom.xml trunk/wikitty-perf-test/src/ trunk/wikitty-perf-test/src/main/ trunk/wikitty-perf-test/src/main/java/ trunk/wikitty-perf-test/src/main/java/org/ trunk/wikitty-perf-test/src/main/java/org/nuiton/ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTest.java trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelAdd.java trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelSearch.java trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestSearch.java trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestUtils.java Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-12-09 11:26:31 UTC (rev 586) +++ trunk/pom.xml 2010-12-09 15:14:22 UTC (rev 587) @@ -30,6 +30,7 @@ <module>wikitty-ui-zk</module> <module>wikitty-hessian-client</module> <module>wikitty-hessian-server</module> + <module>wikitty-perf-test</module> </modules> Added: trunk/wikitty-perf-test/LICENSE.txt =================================================================== --- trunk/wikitty-perf-test/LICENSE.txt (rev 0) +++ trunk/wikitty-perf-test/LICENSE.txt 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,166 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + Added: trunk/wikitty-perf-test/pom.xml =================================================================== --- trunk/wikitty-perf-test/pom.xml (rev 0) +++ trunk/wikitty-perf-test/pom.xml 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.nuiton</groupId> + <artifactId>wikitty</artifactId> + <version>3.0-SNAPSHOT</version> + </parent> + + <groupId>org.nuiton.wikitty</groupId> + <artifactId>wikitty-perf-test</artifactId> + + <dependencies> + +  + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>wikitty-api</artifactId> + <version>${project.version}</version> + </dependency> + +  + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-beans</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>commons-dbcp</groupId> + <artifactId>commons-dbcp</artifactId> + </dependency> + <dependency> + <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>javax.transaction</groupId> + <artifactId>jta</artifactId> + </dependency> + + <dependency> + <groupId>jboss.jbossts</groupId> + <artifactId>jbossjta</artifactId> + </dependency> + + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + </dependency> + + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + </dependency> + </dependencies> + +  +  +  + <name>Wikitty :: wikitty-perf-test</name> + <description>Wikitty performance tests</description> + <inceptionYear>2010</inceptionYear> +</project> \ No newline at end of file Added: trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTest.java =================================================================== --- trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTest.java (rev 0) +++ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTest.java 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,100 @@ +package org.nuiton.wikitty.perftest; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.lang.RandomStringUtils; +import org.nuiton.wikitty.WikittyProxy; +import org.nuiton.wikitty.WikittyConfig; +import org.nuiton.wikitty.WikittyService; +import org.nuiton.wikitty.WikittyServiceFactory; + +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyImpl; +import org.nuiton.wikitty.entities.FieldType.TYPE; +import org.nuiton.wikitty.entities.ExtensionFactory; +import org.nuiton.wikitty.entities.WikittyExtension; + +public class PerfTest { + public static void main(String[] args) { + WikittyConfig config = new WikittyConfig(args); + int threadsNb = 20; + int researches = 2001; + int loopsToAvg = 5; + int wikittiesToCreate = 70; + if (threadsNb > researches) + threadsNb = researches; + boolean skipSearchesTests = false; + boolean skipLabelsTests = false; + + WikittyService ws = WikittyServiceFactory.buildWikittyService(config); + WikittyProxy proxy = new WikittyProxy(ws); + WikittyExtension ext = ExtensionFactory.create("perfTest", "1") + .addField("name", TYPE.STRING).addField("surname", TYPE.STRING) + .addField("age", TYPE.NUMERIC).addField("height", TYPE.NUMERIC) + .addField("birth", TYPE.DATE).addField("wedding", TYPE.DATE) + .addField("isTrue", TYPE.BOOLEAN).addField("isFalse", TYPE.BOOLEAN) + .addField("wiki1", TYPE.WIKITTY).addField("wiki2", TYPE.WIKITTY).extension(); + + Wikitty w = new WikittyImpl(); + Random rand = new Random(); + List<String> ids = new ArrayList<String>(); + for (int i = 0; i < wikittiesToCreate; i++) { + w = new WikittyImpl(); + ids.add(w.getId()); + w.addExtension(ext); + w.setField(ext.getName(), "name", RandomStringUtils.randomAscii(15)); + w.setField(ext.getName(), "surname", RandomStringUtils.randomAscii(20)); + w.setField(ext.getName(), "age", rand.nextInt(110)); + w.setField(ext.getName(), "height", rand.nextInt(230)); + w.setField(ext.getName(), "isTrue", rand.nextInt(1) == 0 ? true : false); + w.setField(ext.getName(), "isFalse", rand.nextInt(1) == 1 ? true : false); + w.setField(ext.getName(), "birth", "01/01/" + rand.nextInt(2010)); + w.setField(ext.getName(), "wiki1", w); + proxy.store(w); + } + + PerfTestUtils.out("Tests on searches :"); + if (!skipSearchesTests) { + PerfTestUtils.out("== Tests on fields of type STRING =="); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "name", "toto", true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "name", "toto", false); + + PerfTestUtils.out("------------\n"); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "surname", "a", true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "surname", "a", false); + + PerfTestUtils.out("== Tests on fields of type NUMERIC =="); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "age", "40", true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "age", "40", false); + + PerfTestUtils.out("------------\n"); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "height", "100", true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "height", "100", false); + + PerfTestUtils.out("== Tests on a field of type BOOLEAN =="); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "isTrue", "true", true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "isTrue", "true", false); + + PerfTestUtils.out("== Tests on a field of type DATE =="); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "birth", "06/07/1720", true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "birth", "06/07/1720", false); + + PerfTestUtils.out("== Tests on a field of type WIKITTY =="); + String wikittySearched = proxy.restore(ids.get(rand.nextInt(wikittiesToCreate))).toString(); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "wiki1", wikittySearched, true); + PerfTestSearch.search(ws, ext, loopsToAvg, researches, wikittiesToCreate, threadsNb, "wiki1", wikittySearched, false); + } else { + PerfTestUtils.out("Skipped"); + } + + PerfTestUtils.out("Tests on labels :"); + if (!skipLabelsTests) { + PerfTestLabelAdd.addLabel(proxy, ids, loopsToAvg, threadsNb); + PerfTestLabelSearch.searchLabel(proxy, ids, loopsToAvg, threadsNb); + } else { + PerfTestUtils.out("Skipped"); + } + } +} Added: trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelAdd.java =================================================================== --- trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelAdd.java (rev 0) +++ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelAdd.java 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,89 @@ +package org.nuiton.wikitty.perftest; + +import java.util.ArrayList; +import java.util.List; +import org.nuiton.wikitty.WikittyProxy; +import org.nuiton.wikitty.addons.WikittyLabelUtil; + +public class PerfTestLabelAdd { + private static class PerfTestLabelAddThread extends Thread { + private int labelsToAdd; + private WikittyProxy proxy; + private List<String> ids; + private int startAt; + + /** + * + * @param labelsToAdd + * @param proxy + * @param ids + * @param startAt + */ + PerfTestLabelAddThread(int labelsToAdd, WikittyProxy proxy, List<String> ids, int startAt) { + this.labelsToAdd = labelsToAdd; + this.proxy = proxy; + this.ids = ids; + this.startAt = startAt; + } + + @Override + public void run() { + long time = System.currentTimeMillis(); + + for (int i = startAt; i < (labelsToAdd + startAt); i++) { + WikittyLabelUtil.addLabel(proxy, ids.get(i), ids.get(i) + time); + } + } + } + + /** + * Adds labels on the wikitties ids and stores them in the proxy + * + * @param proxy : the WikittyProxy to add labels in + * @param ids : the wikitties ids to add labels on + * @param loopsToAvg : the number of times the labels will be added in order to give a more accurate result + * @param threadsNb : the number of threads that will be created to add a part of the labels + */ + public static void addLabel(WikittyProxy proxy, List<String> ids, int loopsToAvg, int threadsNb) { + long time = System.currentTimeMillis(); + int wikittiesToCreate = ids.size(); + + for (int k = 0; k < loopsToAvg; k++) { + for (String id : ids) { + WikittyLabelUtil.addLabel(proxy, id, id + time); + } + } + time = System.currentTimeMillis() - time; + PerfTestUtils.out("1 thread took (average on " + loopsToAvg + " loops) " + (time / loopsToAvg) + " ms to add " + wikittiesToCreate + + " labels on " + wikittiesToCreate + " wikitties\n"); + + List<PerfTestLabelAddThread> threads = new ArrayList<PerfTestLabelAddThread>(); + + time = System.currentTimeMillis(); + int threadMod = ((wikittiesToCreate % threadsNb) != 0 ? 1 : 0); + + for (int j = 0; j < loopsToAvg; j++) { + for (int i = 0; i < threadsNb; i++) { + threads.add(new PerfTestLabelAddThread(wikittiesToCreate / threadsNb, proxy, ids, i * (wikittiesToCreate / threadsNb))); + } + if (threadMod == 1) { + threads.add(new PerfTestLabelAddThread(wikittiesToCreate % threadsNb, proxy, ids, threadsNb * (wikittiesToCreate / threadsNb))); + } + for (int i = 0; i < (threadsNb + threadMod); i++) { + threads.get(i).start(); + } + for (int i = 0; i < (threadsNb + threadMod); i++) { + try { + threads.get(i).join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + threads.clear(); + } + time = System.currentTimeMillis() - time; + PerfTestUtils.out(threadsNb + (threadMod == 1 ? "(+1)" : "") + " threads took (average on " + loopsToAvg + " loops) " + (time / loopsToAvg) + + " ms to add " + (wikittiesToCreate / threadsNb) + " labels each " + (threadMod == 1 ? "(+" + (wikittiesToCreate % threadsNb) + + " labels) " : "") + "on " + wikittiesToCreate + " wikitties\n"); + } +} Added: trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelSearch.java =================================================================== --- trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelSearch.java (rev 0) +++ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestLabelSearch.java 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,101 @@ +package org.nuiton.wikitty.perftest; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.nuiton.wikitty.WikittyProxy; +import org.nuiton.wikitty.addons.WikittyLabelUtil; + +public class PerfTestLabelSearch { + private static class PerfTestLabelSearchThread extends Thread { + private int labelsToSearch; + private WikittyProxy proxy; + private List<String> ids; + private int startAt; + + /** + * + * @param labelsToSearch + * @param proxy + * @param ids + * @param startAt + */ + PerfTestLabelSearchThread(int labelsToSearch, WikittyProxy proxy, List<String> ids, int startAt) { + this.labelsToSearch = labelsToSearch; + this.proxy = proxy; + this.ids = ids; + this.startAt = startAt; + } + + @Override + public void run() { + Set<String> labels = new HashSet<String>(); + + for (int i = startAt; i < (labelsToSearch + startAt); i++) { + labels.addAll(WikittyLabelUtil.findAllAppliedLabels(proxy, ids.get(i))); + } + for (String label : labels) { + WikittyLabelUtil.findAllByLabel(proxy, label, startAt, labelsToSearch + startAt); + } + } + } + + /** + * Searches all labels applied to a wikitty and then retrieves the wikitties associated with these labels + * + * @param proxy : the WikittyProxy to search in + * @param ids : the wikitties ids to search labels on + * @param loopsToAvg : the number of times the searches will be made in order to give a more accurate result + * @param threadsNb : the number of threads that will be created to do a part of the search + */ + public static void searchLabel(WikittyProxy proxy, List<String> ids, int loopsToAvg, int threadsNb) { + long time = System.currentTimeMillis(); + int wikittiesCreated = ids.size(); + Set<String> labels = new HashSet<String>(); + + for (int k = 0; k < loopsToAvg; k++) { + for (String id : ids) { + labels.addAll(WikittyLabelUtil.findAllAppliedLabels(proxy, id)); + } + int size = labels.size(); + + for (String label : labels) { + WikittyLabelUtil.findAllByLabel(proxy, label, 0, size); + } + } + time = System.currentTimeMillis() - time; + PerfTestUtils.out("1 thread took (average on " + loopsToAvg + " loops) " + (time / loopsToAvg) + " ms to find " + labels.size() + + " labels on " + wikittiesCreated + " wikitties\n"); + + List<PerfTestLabelSearchThread> threads = new ArrayList<PerfTestLabelSearchThread>(); + + time = System.currentTimeMillis(); + int threadMod = ((labels.size() % threadsNb) != 0 ? 1 : 0); + + for (int j = 0; j < loopsToAvg; j++) { + for (int i = 0; i < threadsNb; i++) { + threads.add(new PerfTestLabelSearchThread(wikittiesCreated / threadsNb, proxy, ids, i * (wikittiesCreated / threadsNb))); + } + if (threadMod == 1) { + threads.add(new PerfTestLabelSearchThread(wikittiesCreated % threadsNb, proxy, ids, threadsNb * (wikittiesCreated / threadsNb))); + } + for (int i = 0; i < (threadsNb + threadMod); i++) { + threads.get(i).start(); + } + for (int i = 0; i < (threadsNb + threadMod); i++) { + try { + threads.get(i).join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + threads.clear(); + } + time = System.currentTimeMillis() - time; + PerfTestUtils.out(threadsNb + (threadMod == 1 ? "(+1)" : "") + " threads took (average on " + loopsToAvg + " loops) " + + (time / loopsToAvg) + " ms to find " + (labels.size() / threadsNb) + " labels each " + + (threadMod == 1 ? "(+" + (labels.size() % threadsNb) + " labels)" : "") + " on " + wikittiesCreated + " wikitties\n"); + } +} Added: trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestSearch.java =================================================================== --- trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestSearch.java (rev 0) +++ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestSearch.java 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,156 @@ +package org.nuiton.wikitty.perftest; + +import java.util.ArrayList; +import java.util.List; + +import org.nuiton.wikitty.WikittyService; +import org.nuiton.wikitty.entities.Wikitty; +import org.nuiton.wikitty.entities.WikittyExtension; +import org.nuiton.wikitty.search.Criteria; +import org.nuiton.wikitty.search.PagedResult; +import org.nuiton.wikitty.search.Search; +import org.nuiton.wikitty.services.WikittyServiceEnhanced; + +public class PerfTestSearch { + private static class PerfTestSearchThread extends Thread { + private int researches; + private WikittyService ws; + private WikittyExtension ext; + private boolean withRestore; + private String searchIn; + private String searched; + + PerfTestSearchThread(int researches, WikittyService ws, WikittyExtension ext, boolean withRestore, String searchIn, String searched) { + this.researches = researches; + this.ws = ws; + this.ext = ext; + this.withRestore = withRestore; + this.searchIn = searchIn; + this.searched = searched; + } + + @Override + public void run() { + searchGt(researches, ws, ext, searchIn, searched, withRestore); + } + } + + /** + * Searches everything greater than the value `searched` in the field `searchIn` + * + * @param researches : the number of researches to do + * @param ws : the wikitty service + * @param ext : the wikitty extension + * @param searchIn : the field name + * @param searched : the field value to search + * @param withRestore : if the wikitty will be restored after the search + */ + private static void searchGt(int researches, WikittyService ws, WikittyExtension ext, String searchIn, String searched, boolean withRestore) { + for (int i = 0; i < researches; i++) { + Criteria criteria = Search.query().gt(ext.getName() + "." + searchIn, searched).criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + List<String> found = result.getAll(); + for (int j = 0; j < found.size(); j++) { + if (withRestore) { + Wikitty wikittyFound = WikittyServiceEnhanced.restore(ws, null, found.get(j)); + wikittyFound.getFieldAsString(ext.getName(), searchIn); + } + } + } + } + + /** + * Searches with one thread and then displays the result + * + * @param ws : the wikitty service + * @param ext : the wikitty extension + * @param loopsToAvg : the number of times the labels searches will be made in order to give a more accurate result + * @param researches : the number of researches + * @param wikittiesCreated : the number of wikitties in the proxy + * @param searchIn : the field name + * @param searched : the field value to search + * @param withRestore : if the wikitty will be restored after the search + */ + public static void oneThreadSearch(WikittyService ws, WikittyExtension ext, int loopsToAvg, int researches, + int wikittiesCreated, String searchIn, String searched, boolean withRestore) { + long time = System.currentTimeMillis(); + + for (int i = 0; i < loopsToAvg; i++) { + searchGt(researches, ws, ext, searchIn, searched, withRestore); + } + time = System.currentTimeMillis() - time; + if (withRestore) + PerfTestUtils.out("With WikittyServiceEnhanced.restore : "); + else + PerfTestUtils.out("Without WikittyServiceEnhanced.restore : "); + PerfTestUtils.out("1 thread took (average on " + loopsToAvg + " loops) " + (time / loopsToAvg) + " ms to do " + researches + + " researches on field " + searchIn + " in " + wikittiesCreated + " wikitties\n"); + } + + /** + * Searches with n threads and then displays the result + * + * @param ws : the wikitty service + * @param ext : the wikitty extension + * @param loopsToAvg : the number of times the labels searches will be made in order to give a more accurate result + * @param researches : the number of researches + * @param wikittiesCreated : the number of wikitties in the proxy + * @param threadsNb : the number of threads that will be created to do a part of the search + * @param searchIn : the field name + * @param searched : the field value to search + * @param withRestore : if the wikitty will be restored after the search + */ + public static void nThreadsSearch(WikittyService ws, WikittyExtension ext, int loopsToAvg, int researches, + int wikittiesCreated, int threadsNb, String searchIn, String searched, boolean withRestore) { + List<PerfTestSearchThread> threads = new ArrayList<PerfTestSearchThread>(); + long time = System.currentTimeMillis(); + int threadMod = ((researches % threadsNb) != 0 ? 1 : 0); + + for (int j = 0; j < loopsToAvg; j++) { + for (int i = 0; i < threadsNb; i++) { + threads.add(new PerfTestSearchThread(researches / threadsNb, ws, ext, withRestore, searchIn, searched)); + } + if (threadMod == 1) { + threads.add(new PerfTestSearchThread(researches % threadsNb, ws, ext, withRestore, searchIn, searched)); + } + for (int i = 0; i < (threadsNb + threadMod); i++) { + threads.get(i).start(); + } + for (int i = 0; i < (threadsNb + threadMod); i++) { + try { + threads.get(i).join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + threads.clear(); + } + time = System.currentTimeMillis() - time; + if (withRestore) + PerfTestUtils.out("With WikittyServiceEnhanced.restore : "); + else + PerfTestUtils.out("Without WikittyServiceEnhanced.restore : "); + PerfTestUtils.out(threadsNb + (threadMod == 1 ? "(+1)" : "") + " threads took (average on " + loopsToAvg + " loops) " + (time / loopsToAvg) + + " ms to do " + (researches / threadsNb) + " researches each " + (threadMod == 1 ? "(+" + (researches % threadsNb) + " researches) " : "") + + "on field " + searchIn + " in " + wikittiesCreated + " wikitties\n"); + } + + /** + * Launches the searches with one thread and then with n threads + * + * @param ws : the wikitty service + * @param ext : the wikitty extension + * @param loopsToAvg : the number of times the labels searches will be made in order to give a more accurate result + * @param researches : the number of researches + * @param wikittiesCreated : the number of wikitties in the proxy + * @param threadsNb : the number of threads that will be created to do a part of the search + * @param searchIn : the field name + * @param searched : the field value to search + * @param withRestore : if the wikitty will be restored after the search + */ + public static void search(WikittyService ws, WikittyExtension ext, int loopsToAvg, int researches, + int wikittiesCreated, int threadsNb, String searchIn, String searched, boolean withRestore) { + oneThreadSearch(ws, ext, loopsToAvg, researches, wikittiesCreated, searchIn, searched, withRestore); + nThreadsSearch(ws, ext, loopsToAvg, researches, wikittiesCreated, threadsNb, searchIn, searched, withRestore); + } +} Added: trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestUtils.java =================================================================== --- trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestUtils.java (rev 0) +++ trunk/wikitty-perf-test/src/main/java/org/nuiton/wikitty/perftest/PerfTestUtils.java 2010-12-09 15:14:22 UTC (rev 587) @@ -0,0 +1,7 @@ +package org.nuiton.wikitty.perftest; + +public class PerfTestUtils { + public static void out(Object msg) { + System.out.println(msg); + } +}

1 0