Author: bpoussin Date: 2013-12-08 19:57:15 +0100 (Sun, 08 Dec 2013) New Revision: 3870 Url: http://forge.codelutin.com/projects/isis-fish/repository/revisions/3870 Log: doesn't use 'removeStep' to detect step change, use directly 'put' and 'clear' this prevent DefaultSimulator modification which would result in a malfunction of the cache Modified: trunk/src/main/java/fr/ifremer/isisfish/util/IsisCache.java trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java Modified: trunk/src/main/java/fr/ifremer/isisfish/util/IsisCache.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/util/IsisCache.java 2013-12-06 17:32:33 UTC (rev 3869) +++ trunk/src/main/java/fr/ifremer/isisfish/util/IsisCache.java 2013-12-08 18:57:15 UTC (rev 3870) @@ -40,9 +40,9 @@ * <p> * @author poussin * @version $Revision$ - * <p> - * Last update: $Date$ by : - * $Author$ + <p> + Last update: $Date$ by : + $Author$ */ public class IsisCache { @@ -220,6 +220,8 @@ * <p> */ public String printStatistiqueAndClear() { + cacheBackend.clear(); + StringBuilder result = new StringBuilder(); result.append("--- Cache Statistiques ---\n"); result.append("Total call: ").append(totalCall).append("\n"); @@ -231,7 +233,6 @@ result.append("Cache usage: ").append(percent).append("%\n"); result.append(cacheBackend.getStat()); result.append("--------------------\n"); - cacheBackend.clear(); System.out.println(result.toString()); return result.toString(); Modified: trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java =================================================================== --- trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java 2013-12-06 17:32:33 UTC (rev 3869) +++ trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java 2013-12-08 18:57:15 UTC (rev 3870) @@ -3,14 +3,18 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheStats; import com.google.common.cache.Weigher; import fr.ifremer.isisfish.types.TimeStep; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.Writer; import java.util.Collection; import org.apache.commons.lang3.ClassUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.math.matrix.MatrixND; +import org.nuiton.util.StringUtil; /** * Cache qui reserve de la place pour l'equivalent de N pas de temps dans le cache. @@ -86,9 +90,8 @@ } } - backend.totalCached += result; - backend.biggestCached = Math.max(backend.biggestCached, result); - + backend.addStat(result); + result = Math.max(1024, result); // min to 1ko return result; } @@ -98,51 +101,146 @@ protected IsisWeigher isisWeigher; protected long maxMemory; protected long maxCache; - /** le total en octet mis en backend */ + protected long sizeCache; + + /** the last step use to put value in cache */ + protected TimeStep lastStep = null; + /** if true size of cache has been recomputed */ + protected boolean adjusted = false; + + /** le total en octet mis en cache au precedent pas de temps*/ + protected long totalLastStepCached = 0; + + // STATISTIQUE PAR ENTREE DANS LE CACHE + /** le total en d'element mis en cache */ + protected long numberCached = 0; + /** le total en octet mis en cache */ protected long totalCached = 0; - /** la plus grosse donnee mis en backend */ - protected int biggestCached = 0; - protected boolean ajusted = false; + /** la plus petite donnee mis en cache */ + protected long smallestCached = Long.MAX_VALUE; + /** la plus grosse donnee mis en cache */ + protected long biggestCached = 0; + // for variance + double delta = 0; + double mean = 0; + double M2 = 0; + + // STATISTIQUE PAR PAS DE TEMPS + /** le total en d'element mis en cache */ + protected long numberStepCached = 0; + /** le total en octet mis en cache */ + protected long totalStepCached = 0; + /** la plus petite donnee mis en cache */ + protected long smallestStepCached = Long.MAX_VALUE; + /** la plus grosse donnee mis en cache */ + protected long biggestStepCached = 0; + + // for variance + double stepDelta = 0; + double stepMean = 0; + double stepM2 = 0; + + + public IsisCacheBackendOnGuava() { isisWeigher = new IsisWeigher(this); - // TODO poussin 20131117 300Mo and 500Mo must be refined + maxMemory = Runtime.getRuntime().maxMemory(); - maxCache = maxMemory - 300*1024*1024; // isis need about 300M - maxCache = Math.max(maxCache, 500*1024*1024); // to do simulation we need 500M minimum + maxCache = maxMemory - 500*1024*1024; // isis need about 500M + maxCache = Math.max(maxCache, 200*1024*1024); // to do simulation we need 200M cache - // create first cache, this cache is juste for one step (see removeStep) - cache = CacheBuilder.newBuilder() - .maximumWeight(maxCache) - .weigher(isisWeigher) - .build(); + // create first cache, this cache is juste for one step + adjustedCache(maxCache); } + public void addStat(long value) { + numberCached++; + totalCached = totalCached + value; + + // compute variance + delta = value - mean; + mean = mean + delta/numberCached; + M2 = M2 + delta * (value - mean); + + biggestCached = Math.max(biggestCached, value); + smallestCached = Math.min(smallestCached, value); + } + + public void addStepStat(long value) { + numberStepCached++; + totalStepCached = totalStepCached + value; + + // compute variance + stepDelta = value - stepMean; + stepMean = stepMean + stepDelta/numberStepCached; + stepM2 = stepM2 + stepDelta * (value - stepMean); + + biggestStepCached = Math.max(biggestStepCached, value); + smallestStepCached = Math.min(smallestStepCached, value); + } + @Override public void clear() { cache.invalidateAll(); + // clear is assimilate to step change (last step end) + // useful to finalize variance computation for last step + changeStep(); } @Override public void removeStep(Object o) { - // after first step, use information to resize cache - if (!ajusted) { - Cache<String, Object> oldCache = cache; - // compute new size for cache with data from the first step - // if this new size is less than old maxCache, use it - // to keep more memory for JVM - maxCache = Math.min(maxCache, totalCached * CACHE_STEP); // keep place for 3 step - cache = CacheBuilder.newBuilder() - .maximumWeight(maxCache) - .weigher(isisWeigher) - .initialCapacity((int)oldCache.size() * 3) - .build(); + } - log.info(String.format("Cache size ajusted to %sMo (equivalent to %s step need)", - maxCache/1024/1024, CACHE_STEP)); + /** + * resizes the cache if amount is not greater than autorized maxCache + * @param amount new size of cache in byte + */ + public void adjustedCache(long amount){ + // on estime le nombre d'entree necessaire pour ce volume de donnees + long bytePerStep = totalCached / Math.max(1, numberStepCached); + long entriesPerStep = numberCached / Math.max(1, numberStepCached); + + long initialCapacity = amount * entriesPerStep / Math.max(1, bytePerStep); + long stepEquivalent = initialCapacity / entriesPerStep; + + // use amount as new size for cache + // if this new size is less than maxCache, use it + // to keep more memory for JVM + sizeCache = Math.min(this.maxCache, amount); + + CacheBuilder builder = CacheBuilder.newBuilder(); + builder.maximumWeight(sizeCache); + builder.weigher(isisWeigher); + if (initialCapacity > 0) { + builder.initialCapacity((int)initialCapacity); } + cache = builder.build(); + + log.info(String.format("Cache size ajusted to %s (equivalent to %s step need)", + StringUtil.convertMemory(sizeCache), stepEquivalent)); } + /** + * Previent le cache, qu'on change de TimeStep et qu'il peut faire + * du travail s'il en a besoin. + * + * - calcul de statistique + * - ajustement de la taille du cache + */ + public void changeStep() { + // on est passe au pas de temps suivant + long stepAmount = totalCached - totalLastStepCached; + totalLastStepCached = totalCached; + addStepStat(stepAmount); + + // after first step, use information to resize cache + if (!adjusted) { + adjustedCache(stepAmount * CACHE_STEP); // keep place for 3 step + adjusted = true; + } + } + @Override public Object get(TimeStep step, String key) { Object result = cache.getIfPresent(key); @@ -152,23 +250,37 @@ @Override public void put(TimeStep step, String key, Object value) { cache.put(key, value); + if (step != null) { + if (lastStep == null) { + lastStep = step; + } else if (lastStep.getStep() != step.getStep()) { + // if step differ, this indicate new Step + changeStep(); + lastStep = step; + } + } } @Override public String getStat() { - CacheStats stat = cache.stats(); - long evictionCount = stat.evictionCount(); - long hitCount = stat.hitCount(); - double hitRate = stat.hitRate(); - long missCount = stat.missCount(); - double missRate = stat.missRate(); + long mean = totalCached / Math.max(1, numberCached); + long stepMean = totalStepCached / Math.max(1, numberStepCached); + double variance = M2/Math.max(1, numberCached - 1); + double stepVariance = stepM2/Math.max(1, numberStepCached - 1); + + double standardDeviation = Math.sqrt(variance); + double stepStandardDeviation = Math.sqrt(stepVariance); + return String.format("%s - \n" - + "\tmaxMemory: %s, cacheMemory: %s, totalCached: %s, biggestCached: %s\n", -// + "\t evictionCount: %s, hitCount: %s, hitRate: %s, missCount: %s, missRate: %s\n", + + "\tmaxMemory: %s, cacheMemory: %s, totalCached: %s, numberStep: %s, numberEntries: %s\n" + + "\tStep min: %s, max: %s, mean: %s, standard deviation: %s\n" + + "\tEntry min: %s, max: %s, mean: %s, standard deviation: %s\n", getClass().getSimpleName(), - maxMemory, maxCache, totalCached, biggestCached, - evictionCount, hitCount, hitRate, missCount, missRate); + StringUtil.convertMemory(maxMemory), StringUtil.convertMemory(sizeCache), StringUtil.convertMemory(totalCached), numberStepCached, numberCached, + StringUtil.convertMemory(smallestStepCached), StringUtil.convertMemory(biggestStepCached), StringUtil.convertMemory(stepMean), StringUtil.convertMemory((long)stepStandardDeviation), + StringUtil.convertMemory(smallestCached), StringUtil.convertMemory(biggestCached), StringUtil.convertMemory(mean), StringUtil.convertMemory((long)standardDeviation) + ); }