[Isis-fish-commits] r4027 - in trunk: . src/main/java/fr/ifremer/isisfish src/main/java/fr/ifremer/isisfish/datastore src/main/java/fr/ifremer/isisfish/simulator src/main/java/fr/ifremer/isisfish/simulator/launcher src/main/java/fr/ifremer/isisfish/types src/main/java/fr/ifremer/isisfish/ui src/main/java/fr/ifremer/isisfish/ui/simulator src/main/resources/i18n

Author: bpoussin Date: 2014-06-30 19:08:42 +0200 (Mon, 30 Jun 2014) New Revision: 4027 Url: http://forge.codelutin.com/projects/isis-fish/repository/revisions/4027 Log: - permit to configure number of step result to save - add memory cache in ResultStorageCSV with configurable size - ResultStorageInMemory is now deprecated - add some information in param tag/value to transfert configuration to server - add some method in TimeStep (add, minus, gap) - add information about ResultStorage used in simulation Modified: trunk/pom.xml trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorage.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java trunk/src/main/java/fr/ifremer/isisfish/types/TimeStep.java trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java trunk/src/main/java/fr/ifremer/isisfish/ui/simulator/SimulAction.java trunk/src/main/resources/i18n/isis-fish_en_GB.properties trunk/src/main/resources/i18n/isis-fish_fr_FR.properties Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/pom.xml 2014-06-30 17:08:42 UTC (rev 4027) @@ -27,7 +27,7 @@ <dependency> <groupId>org.nuiton</groupId> <artifactId>nuiton-config</artifactId> - <version>3.0-alpha-2</version> + <version>3.0-SNAPSHOT</version> <scope>compile</scope> </dependency> Modified: trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -54,6 +54,7 @@ import fr.ifremer.isisfish.actions.OtherAction; import fr.ifremer.isisfish.actions.SimulationAction; import fr.ifremer.isisfish.actions.VCSAction; +import fr.ifremer.isisfish.datastore.SimulationStorage; import fr.ifremer.isisfish.simulator.SimulationContext; import fr.ifremer.isisfish.simulator.SimulationControl; import fr.ifremer.isisfish.simulator.launcher.InProcessSimulatorLauncher; @@ -63,6 +64,7 @@ import fr.ifremer.isisfish.util.IsisCacheBackend; import fr.ifremer.isisfish.util.IsisCacheBackendOnGuava; import fr.ifremer.isisfish.vcs.VCS; +import org.nuiton.config.OverwriteApplicationConfig; import org.nuiton.math.matrix.DoubleVector; /** @@ -193,10 +195,8 @@ addActionAlias(alias, a.action); } } - } - ////////////////////////////////////////////////// // Methode d'acces aux options ////////////////////////////////////////////////// @@ -831,7 +831,33 @@ saveForUser(); } + /** + * Retourne toutes les configurations specifiques aux simulations. + * Ces configurations sont cotes clients, mais lorsqu'on fait une simulation + * sur un serveur, il faut quelles soient utilisees a la place des valeurs + * du serveur + * + * @return + */ + public Map<String, String> getDefaultSimulationConfig() { + Map<String, String> result = new HashMap<String, String>(); + result.put(Option.SIMULATION_MATRIX_VECTOR_CLASS.key, + getOption(Option.SIMULATION_MATRIX_VECTOR_CLASS.key)); + result.put(Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS.key, + getOption(Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS.key)); + result.put(Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS.key, + getOption(Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS.key)); + result.put(Option.SIMULATION_STORE_RESULT_ON_DISK.key, + getOption(Option.SIMULATION_STORE_RESULT_ON_DISK.key)); + result.put(Option.SIMULATION_STORE_RESULT_CACHE_STEP.key, + getOption(Option.SIMULATION_STORE_RESULT_CACHE_STEP.key)); + + result.put(Option.CACHE_BACKEND_FACTORY_CLASS.key, + getOption(Option.CACHE_BACKEND_FACTORY_CLASS.key)); + + return result; + } /** * @return le dictionnaire des tags par defaut d'une simulation a partir @@ -955,7 +981,8 @@ * @since 4.2.0.2 */ public Class getSimulationMatrixVectorClass() { - Class result = getOptionAsClass(Option.SIMULATION_MATRIX_VECTOR_CLASS.key); + ApplicationConfig config = SimulationContext.get().getConfig(); + Class result = config.getOptionAsClass(Option.SIMULATION_MATRIX_VECTOR_CLASS.key); return result; } @@ -967,7 +994,8 @@ * @since 4.2.0.2 */ public Class getSimulationMatrixVectorSparseClass() { - Class result = getOptionAsClass(Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS.key); + ApplicationConfig config = SimulationContext.get().getConfig(); + Class result = config.getOptionAsClass(Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS.key); return result; } @@ -978,18 +1006,45 @@ * @since 4.2.0.2 */ public int getSimulationMatrixThresholdUseSparse() { - int result = getOptionAsInt(Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS.key); + ApplicationConfig config = SimulationContext.get().getConfig(); + int result = config.getOptionAsInt(Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS.key); return result; } /** + * Indique le nombre de pas qui doivent etre sauve, en partant du dernier pas + * de temps. + * + * @return + * @since 4.2.0.2 + */ + public int getSimulationStoreResultOnDisk() { + ApplicationConfig config = SimulationContext.get().getConfig(); + int result = config.getOptionAsInt(Option.SIMULATION_STORE_RESULT_ON_DISK.key); + return result; + } + + /** + * Indique le nombre de pas qui doivent rester en memoire durant la simulation + * + * @return + * @since 4.2.0.2 + */ + public int getSimulationStoreResultCacheStep() { + ApplicationConfig config = SimulationContext.get().getConfig(); + int result = config.getOptionAsInt(Option.SIMULATION_STORE_RESULT_CACHE_STEP.key); + return result; + } + + /** * Retourne la factory a utilise pour le cache * * @return factory a utilise pour le backend de cache * @since 4.2.1.2 */ public IsisCacheBackend.Factory getCacheBackendFactoryClass() { - IsisCacheBackend.Factory result = getOptionAsObject( + ApplicationConfig config = SimulationContext.get().getConfig(); + IsisCacheBackend.Factory result = config.getOptionAsObject( IsisCacheBackend.Factory.class, Option.CACHE_BACKEND_FACTORY_CLASS.key); return result; } @@ -1030,6 +1085,8 @@ SIMULATION_MATRIX_VECTOR_CLASS("simulation.matrix.vector.class", n("isisfish.config.simulation.matrix.vector.class.description"), DoubleBigVector.class.getName()), SIMULATION_MATRIX_VECTOR_SPARSE_CLASS("simulation.matrix.vector.sparse.class", n("isisfish.config.simulation.matrix.vector.sparse.class.description"), DoubleVector.class.getName()), SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS("simulation.matrix.threshold.use.sparse.class", n("isisfish.config.simulation.matrix.threshold.use.sparse.class.description"), "1000"), + SIMULATION_STORE_RESULT_ON_DISK("simulation.store.result.ondisk", n("isisfish.config.simulation.store.result.ondisk.description"), "-1"), + SIMULATION_STORE_RESULT_CACHE_STEP("simulation.store.result.cachestep", n("isisfish.config.simulation.store.result.cachestep.description"), "13"), /** Nombre maximum de thread de simulation in process. */ SIMULATOR_IN_MAXTHREADS("simulation.in.max.threads", n("isisfish.config.main.simulation.in.max.threads.description"), "1"), Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultMappedStorage.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -50,7 +49,6 @@ import fr.ifremer.isisfish.IsisFishRuntimeException; import fr.ifremer.isisfish.types.TimeStep; import fr.ifremer.isisfish.util.BitUtil; -import java.util.HashMap; /** * Cette classe permet de conserver des résultats de simulation. Elle permet @@ -73,8 +71,8 @@ protected RandomAccessFile raf; protected long offset; - protected Map<TimeStep, Map<String, ResultMapped>> stepNameResults = - new TreeMap<TimeStep, Map<String, ResultMapped>>(); +// protected Map<TimeStep, Map<String, ResultMapped>> stepNameResults = +// new TreeMap<TimeStep, Map<String, ResultMapped>>(); protected Map<String, Map<TimeStep, ResultMapped>> nameStepResults = new TreeMap<String, Map<TimeStep, ResultMapped>>(); @@ -364,18 +362,18 @@ TimeStep step = r.getStep(); String name = r.getName(); - getResult(step).put(name, r); +// getResult(step).put(name, r); getResult(name).put(step, r); } - protected Map<String, ResultMapped> getResult(TimeStep step) { - Map<String, ResultMapped> result = stepNameResults.get(step); - if (result == null) { - result = new TreeMap<String, ResultMapped>(); - stepNameResults.put(step, result); - } - return result; - } +// protected Map<String, ResultMapped> getResult(TimeStep step) { +// Map<String, ResultMapped> result = stepNameResults.get(step); +// if (result == null) { +// result = new TreeMap<String, ResultMapped>(); +// stepNameResults.put(step, result); +// } +// return result; +// } protected Map<TimeStep, ResultMapped> getResult(String name) { Map<TimeStep, ResultMapped> result = nameStepResults.get(name); @@ -390,7 +388,7 @@ protected MatrixND readResult(TimeStep step, String name) { MatrixND result = null; - ResultMapped rm = getResult(step).get(name); + ResultMapped rm = getResult(name).get(step); if (rm != null) { result = rm.getMatrix(); } @@ -398,18 +396,18 @@ return result; } - @Override - protected Map<TimeStep, MatrixND> readResult(String name) { - Map<TimeStep, MatrixND> result = Collections.EMPTY_MAP; - Map<TimeStep, ResultMapped> rms = nameStepResults.get(name); - if (rms != null) { - result = new HashMap<TimeStep, MatrixND>(); - for (Map.Entry<TimeStep, ResultMapped> e : rms.entrySet()) { - result.put(e.getKey(), e.getValue().getMatrix()); - } - } - return result; - } +// @Override +// protected Map<TimeStep, MatrixND> readResult(String name) { +// Map<TimeStep, MatrixND> result = Collections.EMPTY_MAP; +// Map<TimeStep, ResultMapped> rms = nameStepResults.get(name); +// if (rms != null) { +// result = new HashMap<TimeStep, MatrixND>(); +// for (Map.Entry<TimeStep, ResultMapped> e : rms.entrySet()) { +// result.put(e.getKey(), e.getValue().getMatrix()); +// } +// } +// return result; +// } @Override protected void writeResult(TimeStep step, String name, MatrixND mat) { Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorage.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorage.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorage.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -42,6 +42,13 @@ public interface ResultStorage extends SimulationResultListener { /** + * Return some information on result storage. Information depend on + * result storage type + * @return + */ + public String getInfo(); + + /** * Permet de savoir si lorsque l'on ajoutera ce resultat, il sera * sauvé ou non. * Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageAbstract.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -28,7 +28,6 @@ import static org.nuiton.i18n.I18n.t; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -66,6 +65,7 @@ import fr.ifremer.isisfish.simulator.SimulationResultListener; import fr.ifremer.isisfish.types.Month; import fr.ifremer.isisfish.types.TimeStep; +import java.util.LinkedHashMap; import org.nuiton.math.matrix.MatrixFactory; /** @@ -88,6 +88,7 @@ static private Log log = LogFactory.getLog(ResultStorageAbstract.class); protected SimulationStorage simulation = null; + protected EntitySemanticsDecorator decorator; /** result enabled */ transient protected Set<String> enabledResult = null; @@ -96,12 +97,12 @@ * Convertie une entite, month, timestep en string et inversement. * Entity = "TopiaId:Entity.toString" */ - static protected class EntitySemanticsDecorator implements SemanticsDecorator { + static protected class EntitySemanticsDecorator implements SemanticsDecorator<Object, String> { static final private String SEP = ":"; protected TopiaContext tx; /** en cle les representation interne (get) en valeur les valeurs decoree (getKey) */ - protected BidiMap<Object, Object> cache = new DualHashBidiMap<>(); + protected BidiMap<String, Object> cache = new DualHashBidiMap<>(); public EntitySemanticsDecorator() { } @@ -111,39 +112,36 @@ } @Override - public Object decorate(Object internalValue) { + public Object decorate(String internalValue) { Object result = cache.get(internalValue); if (result == null && internalValue != null) { + // on retrouve souvent les memes semantiques dans les matrices + // pour minimiser les chaines en memoire, on prend la representation + // interne avant de l'utiliser comme cle. + internalValue = internalValue.intern(); result = internalValue; - if (internalValue instanceof String) { - // on retrouve souvent les memes semantiques dans les matrices - // pour minimiser les chaines en memoire, on prend la representation - // interne avant de l'utiliser comme cle. - internalValue = ((String)internalValue).intern(); - result = internalValue; - if (StringUtils.startsWith((String)internalValue, Month.class.getName())) { - String val = StringUtils.substringAfter((String)internalValue, SEP); - int monthNumber = Integer.parseInt(val); - result = new Month(monthNumber); - } else if (StringUtils.startsWith((String)internalValue, TimeStep.class.getName())) { - String val = StringUtils.substringAfter((String)internalValue, SEP); - int stepNumber = Integer.parseInt(val); - result = new TimeStep(stepNumber); - } else if (StringUtils.startsWith((String)internalValue, "fr.ifremer.isisfish.entities.")) { - if (tx == null) { - result = StringUtils.substringAfter((String)internalValue, SEP); - } else { - try { - String id = StringUtils.substringBefore((String)internalValue, SEP); - result = tx.findByTopiaId(id); - } catch (TopiaException eee) { - log.info("Fallback use string representation because" - + " i can't decorate (String->Entity): " - + internalValue, eee); - // si on arrive pas a convertir cette fois-ci, on - // renvoi internalValue pour que le undecorate est toutes les infos - // et donc que la prochaine fois on y arrive peut etre - } + if (StringUtils.startsWith(internalValue, Month.class.getName())) { + String val = StringUtils.substringAfter(internalValue, SEP); + int monthNumber = Integer.parseInt(val); + result = new Month(monthNumber); + } else if (StringUtils.startsWith(internalValue, TimeStep.class.getName())) { + String val = StringUtils.substringAfter(internalValue, SEP); + int stepNumber = Integer.parseInt(val); + result = new TimeStep(stepNumber); + } else if (StringUtils.startsWith(internalValue, "fr.ifremer.isisfish.entities.")) { + if (tx == null) { + result = StringUtils.substringAfter(internalValue, SEP); + } else { + try { + String id = StringUtils.substringBefore(internalValue, SEP); + result = tx.findByTopiaId(id); + } catch (TopiaException eee) { + log.info("Fallback use string representation because" + + " i can't decorate (String->Entity): " + + internalValue, eee); + // si on arrive pas a convertir cette fois-ci, on + // renvoi internalValue pour que le undecorate est toutes les infos + // et donc que la prochaine fois on y arrive peut etre } } } @@ -153,8 +151,8 @@ } @Override - public Object undecorate(Object decoratedValue) { - Object result = cache.getKey(decoratedValue); + public String undecorate(Object decoratedValue) { + String result = cache.getKey(decoratedValue); if (result == null && decoratedValue != null) { if (decoratedValue instanceof Month) { result = Month.class.getName() + SEP + ((Month)decoratedValue).getMonthNumber(); @@ -168,7 +166,7 @@ // on retrouve souvent les memes semantiques dans les matrices // pour minimiser les chaines en memoire, on prend la representation // interne avant de l'utiliser comme cle. - cache.put(((String)result).intern(), decoratedValue); + cache.put(result.intern(), decoratedValue); } return result; } @@ -176,20 +174,21 @@ } /** - * Return one result for step and name + * Return one result for step and name. Matrix returned must be undecorated + * (semantics must be string representation of object) * * @param step * @param name * @return */ abstract protected MatrixND readResult(TimeStep step, String name); +// /** +// * Return all available result for name in argument +// * @param name +// * @return +// */ +// abstract protected Map<TimeStep, MatrixND> readResult(String name); /** - * Return all available result for name in argument - * @param name - * @return - */ - abstract protected Map<TimeStep, MatrixND> readResult(String name); - /** * write result * @param step * @param name @@ -217,6 +216,27 @@ return MatrixFactory.getInstance(); } + /** + * Les ResultStorage ne doivent pas etre instancier directement, mais + * recuperer a partir d'un + * {@link fr.ifremer.isisfish.datastore.SimulationStorage#getResultStorage()} + * + * @param simulation storage to get result + */ + public ResultStorageAbstract(SimulationStorage simulation) { + this.simulation = simulation; + this.decorator = new EntitySemanticsDecorator(); + } + + /** + * Return some information on result storage. Information depend on + * result storage type + * @return + */ + public String getInfo() { + return getClass().getSimpleName() + " No more information"; + } + protected MatrixND decorate(MatrixND mat, TopiaContext tx) { MatrixND result = null; if (mat != null) { @@ -229,16 +249,39 @@ } /** - * Les ResultStorage ne doivent pas etre instancier directement, mais - * recuperer a partir d'un - * {@link fr.ifremer.isisfish.datastore.SimulationStorage#getResultStorage()} - * - * @param simulation storage to get result + * Return undecorated version of matrix in argument + * @param mat + * @return new matrix undecorated */ - public ResultStorageAbstract(SimulationStorage simulation) { - this.simulation = simulation; + protected MatrixND undecorate(MatrixND mat) { + MatrixND result = null; + + if (mat != null) { + String name = mat.getName(); + String[] dimNames = mat.getDimensionNames(); + List[] sems = new List[mat.getDimCount()]; + for (int i=0,maxi=sems.length; i<maxi; i++) { + sems[i] = undecorate(mat.getSemantic(i)); + } + result = getMatrixFactory().create(name, sems, dimNames); + result.paste(mat); + } + return result; } + /** + * return new list with undecorate item + * @param l + * @return new list + */ + protected List<String> undecorate(List l) { + ArrayList<String> result = new ArrayList<String>(l.size()); + for (Object o : l) { + result.add(decorator.undecorate(o)); + } + return result; + } + @Override public void delete() { close(); @@ -488,76 +531,68 @@ public MatrixND getMatrix(String name, TopiaContext tx) { log.debug("Get result: " + name); - MatrixND resultMat = null; - Map<TimeStep, MatrixND> results = readResult(name); - // recuperation des resultats qui nous interesse - if (!results.isEmpty()) { - // creation de la liste de date - TimeStep lastStep = getLastStep(); - List<TimeStep> steps = new ArrayList<TimeStep>(); - TimeStep step = new TimeStep(0); + // collect de tous les pas de temps possible + TimeStep lastStep = getLastStep(); + List<TimeStep> steps = new ArrayList<TimeStep>(); + TimeStep step = new TimeStep(0); + steps.add(step); + while (step.before(lastStep)) { + step = step.next(); steps.add(step); - while (step.before(lastStep)) { - step = step.next(); - steps.add(step); - } + } - if (log.isTraceEnabled()) { - log.trace("Steps list : " + steps); + + // collect de toutes les matrices existantes + Map<TimeStep, MatrixND> results = new LinkedHashMap<TimeStep, MatrixND>(); + MatrixND sample = null; + for (TimeStep s : steps) { + MatrixND m = readResult(s, name); + if (m != null) { + results.put(s, m); + sample = m; } + } + // creation des dimensions + String[] dimNames = new String[1 + sample.getDimCount()]; + dimNames[0] = t("isisfish.common.date"); + for (int i = 1; i < dimNames.length; i++) { + dimNames[i] = sample.getDimensionName(i - 1); + } - MatrixND mat = results.values().iterator().next(); + // collect des semantics. +1 pour les dates + List[] sem = new List[1 + sample.getDimCount()]; + sem[0] = steps; - // recuperation des noms des dimensions - String[] dimNames = new String[1 + mat.getDimCount()]; - dimNames[0] = t("isisfish.common.date"); - for (int i = 1; i < dimNames.length; i++) { - dimNames[i] = mat.getDimensionName(i - 1); - } + for (int i = 1; i < sem.length; i++) { + sem[i] = new HashList(); + } - // creation de la semantique pour la matrice resultat. +1 pour les dates - List[] sem = new List[1 + mat.getDimCount()]; - sem[0] = steps; - - for (int i = 1; i < sem.length; i++) { - sem[i] = new HashList(); + for (MatrixND m : results.values()) { + if (m != null) { + for (int s = 0; s < m.getDimCount(); s++) { + sem[s + 1].addAll(m.getSemantic(s)); + } } + } - for (MatrixND mattmp : results.values()) { - if (log.isTraceEnabled()) { - log.trace("Ajout de la semantics: " - + Arrays.asList(mattmp.getSemantics())); - } + // creation de la matrice resultat + MatrixND resultMat = getMatrixFactory().create(name, sem, dimNames); - for (int s = 0; s < mattmp.getDimCount(); s++) { - sem[s + 1].addAll(mattmp.getSemantic(s)); - } - } - if (log.isTraceEnabled()) { - log.trace("La semantique final est: " + Arrays.asList(sem)); - } + // recuperation du resultat pour chaque date de la simulation, de Date(0) à lastDate + for (Map.Entry<TimeStep, MatrixND> result : results.entrySet()) { + TimeStep d = result.getKey(); + MatrixND m = result.getValue(); - // creation de la matrice resultat - resultMat = getMatrixFactory().create(name, sem, dimNames); - - // recuperation du resultat pour chaque date de la simulation, de Date(0) à lastDate - for (Map.Entry<TimeStep, MatrixND> result : results.entrySet()) { - TimeStep d = result.getKey(); - mat = result.getValue(); - // on met ce resultat dans la matrice result si besoin - if (mat != null) { - // on recupere dans la matrice resultat l'endroit on il faut - // mettre la matrice - MatrixND submat = resultMat.getSubMatrix(0, d, 1); - // on met les valeur de mat dans la sous matrice extraite - for (MatrixIterator mi = mat.iterator(); mi.next();) { - submat.setValue( - ArrayUtil.concat(new Object[] { d }, + // on recupere dans la matrice resultat l'endroit on il faut + // mettre la matrice + MatrixND submat = resultMat.getSubMatrix(0, d, 1); + // on met les valeur de mat dans la sous matrice extraite + for (MatrixIterator mi = m.iterator(); mi.next();) { + submat.setValue( + ArrayUtil.concat(new Object[] { d }, mi.getSemanticsCoordinates()), mi.getValue()); - } - } } } Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -1,7 +1,9 @@ package fr.ifremer.isisfish.datastore; +import fr.ifremer.isisfish.IsisFish; import fr.ifremer.isisfish.IsisFishRuntimeException; +import fr.ifremer.isisfish.simulator.SimulationContext; import fr.ifremer.isisfish.types.TimeStep; import fr.ifremer.isisfish.util.SimpleParser; import java.io.BufferedWriter; @@ -18,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.zip.GZIPInputStream; @@ -31,9 +34,14 @@ import org.nuiton.math.matrix.MatrixND; /** - * Ecrit les resultats dans des fichiers au format pseudo csv compresser (gz) - * Seule les valeurs differentes de 0 sont ecrite dans le fichier + * Ecrit les resultats dans des fichiers au format pseudo csv compresse (gz) + * Seules les valeurs differentes de 0 sont ecrites dans le fichier. + * + * Il est possible de modifier via la configuration: + * - le nombre de TimeStep en cache + * - de definir le nombre de TimeStep a conserver sur disque * + * * Format d'une matrice: * <pre> * # commentaire @@ -75,13 +83,47 @@ static final private Charset charset = Charset.forName("UTF-8"); - protected EntitySemanticsDecorator decorator; + /** number of step result to store on disk. 0 for none, negative for all result */ + protected int storeOnDiskStep = -1; + /** number of step in cache */ + protected int cacheStep = 13; // default to 13 months + /** nombre total de resultat */ + protected int numberOfResult = 0; + /** nombre de resultat reellement stocke sur disque */ + protected int numberOfResultOnDisk = 0; + /** le nombre de resultat demande qui n'etait pas dans le pas de temps courant */ + protected int numberOfPastResultAsked = 0; + /** le plus vieux pas de temps demande par rapport au pas de temps courant */ + protected int maxGapWithCurrentStep = 0; + + LinkedHashMap<TimeStep, Map<String, MatrixND>> cache = + new LinkedHashMap<TimeStep, Map<String, MatrixND>>() { + private static final long serialVersionUID = 1L; + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > cacheStep; + } + }; + + public ResultStorageCSV(SimulationStorage simulation) { super(simulation); - decorator = new EntitySemanticsDecorator(); + + storeOnDiskStep = IsisFish.config.getSimulationStoreResultOnDisk(); + cacheStep = IsisFish.config.getSimulationStoreResultCacheStep(); } + @Override + public String getInfo() { + return String.format( + "ResultStorageCSV:\n" + + "\t%s results written to disk on a total of %s\n" + + "\t%s results read outside of current step\n" + + "\tthe oldest step was %s step before current step\n", + numberOfResultOnDisk, numberOfResult, numberOfPastResultAsked, maxGapWithCurrentStep); + } + /** * donne le repertoire de stockage des resultats * @return @@ -106,10 +148,6 @@ return file; } - protected MatrixND readMatrix(File file) throws IOException { - return readMatrix(file.getPath()); - } - protected MatrixND readMatrix(String file) throws IOException { MatrixND result = null; LineNumberReader in = null; @@ -195,90 +233,145 @@ @Override protected MatrixND readResult(TimeStep step, String name) { - File file = getMatrixFile(step, name); + // un peu de statistique + TimeStep currentStep = SimulationContext.get().getSimulationControl().getStep(); + if (!step.equals(currentStep)) { + numberOfPastResultAsked++; + maxGapWithCurrentStep = Math.max(maxGapWithCurrentStep, currentStep.gap(step)); + } + MatrixND result = null; - if (file.exists()) { - try { - result = readMatrix(file); - } catch (IOException eee) { - log.error("Can't read result file: " + file, eee); + + // try to get in cache first + if(cache.containsKey(step)) { + result = getCacheForStep(step).get(name); + } + + // if not found looking for result on disk + if (result == null) { + File file = getMatrixFile(step, name); + if (file.exists()) { + try { + result = readMatrix(file.getPath()); + } catch (IOException eee) { + log.error("Can't read result file: " + file, eee); + } } } return result; } - @Override - protected Map<TimeStep, MatrixND> readResult(String name) { - File dir = getMatrixDirectory(name); - String[] files = dir.list(); +// @Override +// protected Map<TimeStep, MatrixND> readResult(String name) { +// File dir = getMatrixDirectory(name); +// String[] files = dir.list(); +// +// Map<TimeStep, MatrixND> result = new HashMap<TimeStep, MatrixND>(); +// for(String file : files) { +// try { +// TimeStep step = getTimeStep(file); +// if (step != null) { +// MatrixND mat = readResult(step, name); +// if (mat != null) { +// result.put(step, mat); +// } +// } +// } catch (Exception eee) { +// log.error("Can't read result file: " + file, eee); +// } +// } +// return result; +// } - Map<TimeStep, MatrixND> result = new HashMap<TimeStep, MatrixND>(); - for(String file : files) { - try { - TimeStep step = getTimeStep(file); - if (step != null) { - MatrixND mat = readMatrix(dir + File.separator + file); - if (mat != null) { - result.put(step, mat); - } - } - } catch (Exception eee) { - log.error("Can't read result file: " + file, eee); - } + /** Le pas de temps a partir duquel il faut sauver les resultats sur disque*/ + protected TimeStep startDiskStep = null; + + /** + * Indique s'il faut sauver sur disque les resultats + * @param step + * @return + */ + protected boolean isDiskResult(TimeStep step) { + if (startDiskStep == null) { + startDiskStep = getLastStep().minus(storeOnDiskStep); } + return storeOnDiskStep < 0 || startDiskStep.before(step); + } + + protected Map<String, MatrixND> getCacheForStep(TimeStep step) { + Map<String, MatrixND> result = cache.get(step); + if (result == null) { + result = new HashMap<String, MatrixND>(); + cache.put(step, result); + } return result; } + @Override protected void writeResult(TimeStep step, String name, MatrixND mat) { - File file = getMatrixFile(step, name); - file.getParentFile().mkdirs(); - PrintWriter out = null; - try { - out = new PrintWriter(new BufferedWriter(new OutputStreamWriter( - new GZIPOutputStream(new FileOutputStream(file)), - charset - ))); + // copy and undecorate matrix + mat = undecorate(mat); + + // add mat in cache + getCacheForStep(step).put(name, mat); - out.print(name); - out.println(); + // for statistics + numberOfResult++; - List[] sems = mat.getSemantics(); - for (int i=0, maxi=sems.length; i<maxi; i++) { - String dimName = mat.getDimensionName(i); - out.print(dimName); + // write mat on disk + if (isDiskResult(step)) { + // for statistics + numberOfResultOnDisk++; + + File file = getMatrixFile(step, name); + file.getParentFile().mkdirs(); + PrintWriter out = null; + try { + out = new PrintWriter(new BufferedWriter(new OutputStreamWriter( + new GZIPOutputStream(new FileOutputStream(file)), + charset + ))); - List l = sems[i]; - char sep = ':'; - for (Object o : l) { - out.print(sep); - out.print(decorator.undecorate(o)); - sep = ';'; - } + out.print(name); out.println(); - } - // blank line to separate header and data - out.println(); + List[] sems = mat.getSemantics(); + for (int i=0, maxi=sems.length; i<maxi; i++) { + String dimName = mat.getDimensionName(i); + out.print(dimName); - for (MatrixIterator i = mat.iteratorNotZero(); i.hasNext();) { - i.next(); - int[] pos = i.getCoordinates(); - double value = i.getValue(); - for (int p : pos) { - out.print(p); - out.print(';'); + List l = sems[i]; + char sep = ':'; + for (Object o : l) { + out.print(sep); + out.print(o); + sep = ';'; + } + out.println(); } - out.print(value); + + // blank line to separate header and data out.println(); + + for (MatrixIterator i = mat.iteratorNotZero(); i.hasNext();) { + i.next(); + int[] pos = i.getCoordinates(); + double value = i.getValue(); + for (int p : pos) { + out.print(p); + out.print(';'); + } + out.print(value); + out.println(); + } + + } catch (Exception eee) { + throw new IsisFishRuntimeException("Can't write result: " + file, eee); + } finally { + IOUtils.closeQuietly(out); } - - } catch (Exception eee) { - throw new IsisFishRuntimeException("Can't write result: " + file, eee); - } finally { - IOUtils.closeQuietly(out); } - } @Override Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageInMemory.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -52,6 +52,8 @@ * * Mise a jour: $Date$ * par : $Author$ + * + * @deprecated ResultStorageCSV now support no storage and in memory usage */ public class ResultStorageInMemory extends ResultStorageAbstract { @@ -103,7 +105,7 @@ @Override protected void writeResult(TimeStep step, String name, MatrixND mat) { - MatrixND newMat = mat.copy(); + MatrixND newMat = undecorate(mat); Map<String, MatrixND> mats = data.get(step); if (mats == null) { mats = new HashMap<String, MatrixND>(); @@ -152,21 +154,21 @@ return result; } - /* (non-Javadoc) - * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(java.lang.String) - */ - @Override - public Map<TimeStep, MatrixND> readResult(String name) { - Map<TimeStep, MatrixND> result = new LinkedHashMap<TimeStep, MatrixND>(); - for (Map.Entry<TimeStep, Map<String, MatrixND>> e : data.entrySet()) { - TimeStep ts = e.getKey(); - MatrixND mat = e.getValue().get(name); - if (mat != null) { - result.put(ts, mat); - } - } +// /* (non-Javadoc) +// * @see fr.ifremer.isisfish.datastore.ResultStorage#getMatrix(java.lang.String) +// */ +// @Override +// public Map<TimeStep, MatrixND> readResult(String name) { +// Map<TimeStep, MatrixND> result = new LinkedHashMap<TimeStep, MatrixND>(); +// for (Map.Entry<TimeStep, Map<String, MatrixND>> e : data.entrySet()) { +// TimeStep ts = e.getKey(); +// MatrixND mat = e.getValue().get(name); +// if (mat != null) { +// result.put(ts, mat); +// } +// } +// +// return result; +// } - return result; - } - } Modified: trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationContext.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -43,6 +43,8 @@ import fr.ifremer.isisfish.types.TimeStep; import fr.ifremer.isisfish.util.IsisCache; import fr.ifremer.isisfish.util.Trace; +import org.nuiton.config.ApplicationConfig; +import org.nuiton.config.OverwriteApplicationConfig; import org.nuiton.math.matrix.MatrixFactory; /** @@ -69,6 +71,8 @@ /** to use log facility, just put in your code: log.info(\"...\"); */ private static Log log = LogFactory.getLog(SimulationContext.class); + /** configuration for current simulation (some value of config is overwrited for simulation) */ + protected ApplicationConfig config; protected Map<String, Object> values = new HashMap<String, Object>(); protected SimulationStorage simulation = null; protected SimulationControl simulationControl = null; @@ -103,7 +107,17 @@ }; protected SimulationContext() { - // force matrix backend for this simulation + } + + /** + * init SimulationContext for simulation. + * + */ + public void initForSimulation() { + // must be done after setSimulationStorage because config looking for + // value in simulation parameter via SimulationContext + + // force matrix backend for this simulationcontext MatrixFactory.initMatrixFactoryThreadLocal( IsisFish.config.getSimulationMatrixVectorClass(), IsisFish.config.getSimulationMatrixVectorSparseClass(), @@ -111,6 +125,8 @@ } /** + * Return SimulationContext if not created, create new context without + * initialisation needed for simulation * @return simulation context for the current simulation (current thread) */ public static SimulationContext get() { @@ -132,6 +148,27 @@ } /** + * Get specifique ApplicationConfig for current thread.simulation. + * Needed for simulation some configurations values are overwrited by + * simulation parameter tag/value + * + * @return + */ + public ApplicationConfig getConfig() { + if (config == null) { + if (getSimulationStorage()!= null) { + // in simulation, tag/value must be in config for simulation option + Map<String, String> tv = getSimulationStorage().getParameter().getTagValue(); + config = new OverwriteApplicationConfig(IsisFish.config, tv); + } else { + config = IsisFish.config; + } + } + return config; + } + + + /** * Add simulation listener, if listener is {@link SimulationResultListener}, it's * automatically added as listener on {@link ResultManager} * @param l Modified: trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/InProcessSimulatorLauncher.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -354,6 +354,7 @@ // Creation et initialisation du context de simulation // SimulationContext context = SimulationContext.get(); + context.setSimulationControl(control != null ? control : new SimulationControl(simulation.getName())); @@ -372,7 +373,7 @@ // is simulation context context.setScriptDirectory(rootDirectory); context.setSimulationStorage(simulation); - + // Warning : Rule have to be instanciated after aspect definition classLoader.deploy(RuleAspect.class); @@ -381,6 +382,10 @@ // forceReload, save all modification in parameter and reread it parameters = simulation.getForceReloadParameter(); + // must be done after setSimulationStorage because config looking for + // value in simulation parameter via SimulationContext + context.initForSimulation(); + // // Activation de l'OAP demandée // @@ -425,9 +430,8 @@ String matrixBackend = MatrixFactory.getInstance().getVectorClass().getName(); String matrixSparseBackend = MatrixFactory.getInstance().getSparseVectorClass().getName(); int threshold = MatrixFactory.getInstance().getThresholdSparse(); - simulation.getInformation().addInformation("Matrix backend: " + log.info("Matrix backend: " + matrixBackend + " and " + matrixSparseBackend + " threshold: " + threshold); - log.info("Matrix backend: " + matrixBackend); simulatorObject.simulate(context); // @@ -465,6 +469,7 @@ if (parameters.isSensitivityAnalysisOnlyKeepFirst() && !control.getId().endsWith("_0")) { resultStorage.delete(); } + simulation.getInformation().addInformation(resultStorage.getInfo()); resultStorage.close(); } catch (OutOfMemoryError eee) { Modified: trunk/src/main/java/fr/ifremer/isisfish/types/TimeStep.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/types/TimeStep.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/types/TimeStep.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -85,6 +85,34 @@ } /** + * Return new TimeStep equals to this.step + number + * @param number number of step to add + * @return new TimeStep + */ + public TimeStep add(int number) { + return new TimeStep(step + number); + } + + /** + * Return new TimeStep equals to this.step - number + * @param number number of step to remove + * @return new TimeStep + */ + public TimeStep minus(int number) { + return new TimeStep(step - number); + } + + /** + * Return number of step between current TimeStep and other TimeStep in argument + * (this - other) + * @param other + * @return + */ + public int gap(TimeStep other) { + return step - other.step; + } + + /** * Method next retourne une nouvelle date qui est la date suivante de la * date courante. la date courante n'est pas modifier. * Modified: trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/ui/WelcomeHandler.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -229,6 +229,8 @@ modelBuilder.addOption(IsisConfig.Option.SIMULATION_MATRIX_VECTOR_SPARSE_CLASS); modelBuilder.addOption(IsisConfig.Option.SIMULATION_MATRIX_THRESHOLD_USE_SPARSE_CLASS); modelBuilder.addOption(IsisConfig.Option.MAPPED_RESULT_MATRIX_VECTOR_CLASS); + modelBuilder.addOption(IsisConfig.Option.SIMULATION_STORE_RESULT_ON_DISK); + modelBuilder.addOption(IsisConfig.Option.SIMULATION_STORE_RESULT_CACHE_STEP); // category database vcs modelBuilder.addCategory(t("isisfish.config.category.officialvcs"), t("isisfish.config.category.officialvcs.description")); Modified: trunk/src/main/java/fr/ifremer/isisfish/ui/simulator/SimulAction.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/ui/simulator/SimulAction.java 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/java/fr/ifremer/isisfish/ui/simulator/SimulAction.java 2014-06-30 17:08:42 UTC (rev 4027) @@ -99,6 +99,8 @@ import fr.ifremer.isisfish.ui.sensitivity.wizard.SensitivityWizardHandler; import fr.ifremer.isisfish.ui.util.ErrorHelper; import fr.ifremer.isisfish.ui.widget.editor.ScriptParameterDialog; +import java.util.HashMap; +import java.util.LinkedHashMap; /** * SimulAction. @@ -152,8 +154,12 @@ List<String> resultNames = getResultNames(); // put default value in param param.setSimulatorName(IsisFish.config.getSimulatorClassfile()); - param.setTagValue(IsisFish.config.getDefaultTagValueAsMap()); + Map<String, String> tv = new LinkedHashMap<String, String>(); + tv.putAll(IsisFish.config.getDefaultTagValueAsMap()); + tv.putAll(IsisFish.config.getDefaultSimulationConfig()); + param.setTagValue(tv); + param.setExportNames(IsisFish.config.getDefaultExportNamesAsList()); List<String> defaultResultNames = IsisFish.config .getDefaultResultNamesAsList(); @@ -987,6 +993,26 @@ return SimulationService.getService().getSimulationLaunchers(); } + protected boolean checkAndPrepare(String fullSimulationId) { + boolean result = !("".equals(fullSimulationId) + || SimulationStorage.localyExists(fullSimulationId) + || SimulationService.getService().exists(fullSimulationId)); + + if (result) { + // force all necessary config simulation in tag value + Map<String, String> m = new HashMap<String, String>( + IsisFish.config.getDefaultSimulationConfig()); + Map<String, String> tv = param.getTagValue(); + m.putAll(tv); + param.setTagValue(m); + } else { + ErrorHelper.showErrorDialog(t("isisfish.simulator.simulaction.badid", + fullSimulationId), null); + } + + return result; + } + /** * Launch automatically the simulation, when is possible (no other simulation) * or wait for the last automatically simulation ended. @@ -1008,12 +1034,7 @@ } try { - if ("".equals(fullSimulationId) - || SimulationStorage.localyExists(fullSimulationId) - || SimulationService.getService().exists(fullSimulationId)) { - ErrorHelper.showErrorDialog(t("isisfish.simulator.simulaction.badid", - fullSimulationId), null); - } else { + if (checkAndPrepare(fullSimulationId)) { SimulationService.getService().submit(fullSimulationId, param, launcher, 0); } @@ -1051,12 +1072,7 @@ } try { - if ("".equals(fullSimulationId) - || SimulationStorage.localyExists(fullSimulationId) - || SimulationService.getService().exists(fullSimulationId)) { - ErrorHelper.showErrorDialog(t("isisfish.simulator.simulaction.badid", - fullSimulationId), null); - } else { + if (checkAndPrepare(fullSimulationId)) { DesignPlan designPlan = new DesignPlan(); designPlan.setFactorGroup(factorGroup); SimulationService.getService().submit(fullSimulationId, param, Modified: trunk/src/main/resources/i18n/isis-fish_en_GB.properties =================================================================== --- trunk/src/main/resources/i18n/isis-fish_en_GB.properties 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/resources/i18n/isis-fish_en_GB.properties 2014-06-30 17:08:42 UTC (rev 4027) @@ -229,6 +229,11 @@ isisfish.config.main.userMail.description=User email address isisfish.config.main.userName.description=User name isisfish.config.mapped.result.matrix.vector.class.description= +isisfish.config.simulation.matrix.threshold.use.sparse.class.description= +isisfish.config.simulation.matrix.vector.class.description= +isisfish.config.simulation.matrix.vector.sparse.class.description= +isisfish.config.simulation.store.result.cachestep.description= +isisfish.config.simulation.store.result.ondisk.description= isisfish.config.ssh.key.file=SSH key isisfish.config.ssh.key.file.description=Private SSH key path isisfish.config.title=Preferences Modified: trunk/src/main/resources/i18n/isis-fish_fr_FR.properties =================================================================== --- trunk/src/main/resources/i18n/isis-fish_fr_FR.properties 2014-06-29 18:44:00 UTC (rev 4026) +++ trunk/src/main/resources/i18n/isis-fish_fr_FR.properties 2014-06-30 17:08:42 UTC (rev 4027) @@ -229,6 +229,11 @@ isisfish.config.main.userMail.description=le courriel de l'utilisateur isisfish.config.main.userName.description=le nom - prénom de l'utilisateur isisfish.config.mapped.result.matrix.vector.class.description= +isisfish.config.simulation.matrix.threshold.use.sparse.class.description= +isisfish.config.simulation.matrix.vector.class.description= +isisfish.config.simulation.matrix.vector.sparse.class.description= +isisfish.config.simulation.store.result.cachestep.description= +isisfish.config.simulation.store.result.ondisk.description= isisfish.config.ssh.key.file=Clé SSH isisfish.config.ssh.key.file.description=Emplacement de la clé privée SSH isisfish.config.title=Préférences