- Isis-fish-commits - mailman.cloud.codelutin.com

r2829 - isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org Dec. 14, 2009

Dec. 14, 2009

Author: chatellier Date: 2009-12-14 15:51:06 +0000 (Mon, 14 Dec 2009) New Revision: 2829 Modified: isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SimulationServiceTest.java Log: Fix test, with JUnit timeout configuration. Modified: isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SimulationServiceTest.java =================================================================== --- isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SimulationServiceTest.java 2009-12-14 15:50:29 UTC (rev 2828) +++ isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SimulationServiceTest.java 2009-12-14 15:51:06 UTC (rev 2829) @@ -23,11 +23,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.math.matrix.MatrixFactory; -import org.nuiton.math.matrix.MatrixND; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; +import org.nuiton.math.matrix.MatrixFactory; +import org.nuiton.math.matrix.MatrixND; import fr.ifremer.isisfish.AbstractIsisFishTest; import fr.ifremer.isisfish.entities.Population; @@ -51,11 +50,7 @@ * @version $Revision: 1.0 $ * * Last update : $Date: 24 févr. 2009 $ By : $Author: chatellier $ - * - * - * FIXME EC-20090713 ignored : test fail and loop infinitely */ - at Ignore public class SimulationServiceTest extends AbstractIsisFishTest { /** Commons logging log. */ @@ -83,8 +78,7 @@ domain1.setMaxBound(50.0); domain1.setCardinality(4); factor1.setDomain(domain1); - factor1 - .setPath("fr.ifremer.isisfish.entities.PopulationGroup#1156461521013#0.1715620681984218#maxLength"); + factor1.setPath("fr.ifremer.isisfish.entities.PopulationGroup#1156461521013#0.1715620681984218#maxLength"); // factor 2 /*Factor<Double> factor2 = new Factor<Double>("factor 2 (double)"); @@ -104,8 +98,7 @@ domain3.setMaxBound(99.0); domain3.setCardinality(4); factor3.setDomain(domain3); - factor3 - .setPath("fr.ifremer.isisfish.entities.PopulationGroup#1156461521076#0.6526656643346673#minLength"); + factor3.setPath("fr.ifremer.isisfish.entities.PopulationGroup#1156461521076#0.6526656643346673#minLength"); designPlan.getFactors().add(factor1); //designPlan.getFactors().add(factor2); @@ -148,8 +141,7 @@ domain1.getValues().put(0.0, matrix1); domain1.getValues().put(1.0, matrix2); factor1.setDomain(domain1); - factor1 - .setPath("fr.ifremer.isisfish.entities.StrategyMonthInfo#1156808754768#0.7282750856395208#proportionMetier"); + factor1.setPath("fr.ifremer.isisfish.entities.StrategyMonthInfo#1156808754768#0.7282750856395208#proportionMetier"); designPlan.getFactors().add(factor1); @@ -158,8 +150,10 @@ /** * Lance une simulation avec des facteurs de sensibilité. + * + * Test is wrong if it take more than 2 min. */ - @Test + @Test(timeout=120 * 1000) public void testRunSensivitySimulation() { SimulationParameter params = new SimulationParameter(); @@ -209,8 +203,10 @@ * Lance une simulation avec des facteurs de sensibilité. * * This test use factors with matrix. + * + * Fail after 2 min. */ - @Test + @Test(timeout=120 * 1000) public void testRunSensivitySimulationWithMatrix() { SimulationParameter params = new SimulationParameter();

1 0

r2828 - in isis-fish/trunk/src/test/resources/test-database: scripts simulators
by chatellier＠users.labs.libre-entreprise.org Dec. 14, 2009

Dec. 14, 2009

Author: chatellier Date: 2009-12-14 15:50:29 +0000 (Mon, 14 Dec 2009) New Revision: 2828 Modified: isis-fish/trunk/src/test/resources/test-database/scripts/SiMatrix.java isis-fish/trunk/src/test/resources/test-database/simulators/DefaultSimulator.java Log: Fix test database compilation Modified: isis-fish/trunk/src/test/resources/test-database/scripts/SiMatrix.java =================================================================== --- isis-fish/trunk/src/test/resources/test-database/scripts/SiMatrix.java 2009-12-14 14:28:56 UTC (rev 2827) +++ isis-fish/trunk/src/test/resources/test-database/scripts/SiMatrix.java 2009-12-14 15:50:29 UTC (rev 2828) @@ -19,7 +19,7 @@ package scripts; -import static org.codelutin.i18n.I18n.n_; +import static org.nuiton.i18n.I18n.n_; import java.util.ArrayList; import java.util.Collection; @@ -29,11 +29,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.codelutin.math.matrix.MatrixFactory; -import org.codelutin.math.matrix.MatrixIterator; -import org.codelutin.math.matrix.MatrixND; -import org.codelutin.topia.TopiaContext; -import org.codelutin.topia.TopiaException; +import org.nuiton.math.matrix.MatrixFactory; +import org.nuiton.math.matrix.MatrixIterator; +import org.nuiton.math.matrix.MatrixND; +import org.nuiton.topia.TopiaContext; +import org.nuiton.topia.TopiaException; import fr.ifremer.isisfish.IsisFishDAOHelper; import fr.ifremer.isisfish.IsisFishException; @@ -61,15 +61,15 @@ * Created: 21 aout 2006 15:53:01 * * @author poussin - * @version $Revision: 1.18 $ + * @version $Revision: 1 $ * - * Last update: $Date: 2007-11-02 17:53:20 $ + * Last update: $Date: 2009-10-19 14:25:49 +0200 (lun., 19 oct. 2009) $ * by : $Author: bpoussin $ */ public class SiMatrix { /** to use log facility, just put in your code: log.info("..."); */ - static private Log log = LogFactory.getLog(SiMatrix.class); + private Log log = LogFactory.getLog(SiMatrix.class); protected SimulationContext context = null; protected TopiaContext db = null; @@ -269,7 +269,7 @@ List<Strategy> strategies = getStrategies(date); List<Metier> metiers = getMetiers(date); - List<PopulationGroup> groups = matrixCatchPerStrategyMetPerCell + List<PopulationGroup> groups = (List<PopulationGroup>)matrixCatchPerStrategyMetPerCell .getSemantics(2); List<Zone> zones = getZones(date); @@ -1361,7 +1361,7 @@ double M = group.getNaturalDeathRate(zone) / (double) Month.NUMBER_OF_MONTH; double value = (double) Math.exp(-M); - ; + double n = N.getValue(g, z); value *= n; result.setValue(g, z, value); @@ -1425,8 +1425,8 @@ * @return */ public MatrixND matrixBiomass(MatrixND N, Population pop, Date date) { - List<PopulationGroup> groups = N.getSemantics(0); - List<Zone> zones = N.getSemantics(1); + List<PopulationGroup> groups = (List<PopulationGroup>)N.getSemantics(0); + List<Zone> zones = (List<Zone>)N.getSemantics(1); MatrixND result = MatrixFactory.getInstance().create( ResultName.MATRIX_BIOMASS, new List[] { groups, zones }, @@ -1448,8 +1448,8 @@ public MatrixND matrixBiomassBeginMonth(MatrixND N, Population pop, Date date) { - List<PopulationGroup> groups = N.getSemantics(0); - List<Zone> zones = N.getSemantics(1); + List<PopulationGroup> groups = (List<PopulationGroup>)N.getSemantics(0); + List<Zone> zones = (List<Zone>)N.getSemantics(1); MatrixND result = MatrixFactory.getInstance().create( ResultName.MATRIX_BIOMASS_BEGIN_MONTH, new List[] { groups, zones }, @@ -1470,8 +1470,8 @@ public MatrixND matrixAbondanceBeginMonth(MatrixND N, Population pop, Date date) { - List<PopulationGroup> groups = N.getSemantics(0); - List<Zone> zones = N.getSemantics(1); + List<PopulationGroup> groups = (List<PopulationGroup>)N.getSemantics(0); + List<Zone> zones = (List<Zone>)N.getSemantics(1); MatrixND result = MatrixFactory.getInstance().create( ResultName.MATRIX_ABUNDANCE_BEGIN_MONTH, new List[] { groups, zones }, Modified: isis-fish/trunk/src/test/resources/test-database/simulators/DefaultSimulator.java =================================================================== --- isis-fish/trunk/src/test/resources/test-database/simulators/DefaultSimulator.java 2009-12-14 14:28:56 UTC (rev 2827) +++ isis-fish/trunk/src/test/resources/test-database/simulators/DefaultSimulator.java 2009-12-14 15:50:29 UTC (rev 2828) @@ -1,6 +1,6 @@ /* *##% - * Copyright (C) 2006 - * Ifremer, Code Lutin, C?dric Pineau, Benjamin Poussin + * Copyright (C) 2006 - 2009 + * Ifremer, Code Lutin, Cédric Pineau, Benjamin Poussin * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -23,26 +23,25 @@ * Created: 21 ao?t 2006 10:57:46 * * @author poussin - * @version $Revision: 1.19 $ + * @version $Revision: 1 $ * - * Last update: $Date: 2007-11-06 18:16:14 $ + * Last update: $Date: 2009-10-19 14:25:49 +0200 (lun., 19 oct. 2009) $ * by : $Author: bpoussin $ */ package simulators; -import static org.codelutin.i18n.I18n._; -import static org.codelutin.i18n.I18n.n_; +import static org.nuiton.i18n.I18n._; +import static org.nuiton.i18n.I18n.n_; -import java.util.Collection; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.codelutin.math.matrix.MatrixFactory; -import org.codelutin.math.matrix.MatrixND; -import org.codelutin.topia.TopiaContext; -import org.codelutin.topia.TopiaException; +import org.nuiton.math.matrix.MatrixFactory; +import org.nuiton.math.matrix.MatrixND; +import org.nuiton.topia.TopiaContext; +import org.nuiton.topia.TopiaException; import scripts.GravityModel; import scripts.ResultName; @@ -52,8 +51,6 @@ import fr.ifremer.isisfish.entities.Metier; import fr.ifremer.isisfish.entities.Population; import fr.ifremer.isisfish.entities.PopulationSeasonInfo; -import fr.ifremer.isisfish.entities.Strategy; -import fr.ifremer.isisfish.entities.Zone; import fr.ifremer.isisfish.rule.Rule; import fr.ifremer.isisfish.simulator.MetierMonitor; import fr.ifremer.isisfish.simulator.PopulationMonitor; @@ -68,43 +65,47 @@ /** * @author poussin - * + * */ public class DefaultSimulator implements Simulator { - + /** to use log facility, just put in your code: log.info("..."); */ - static private Log log = LogFactory.getLog(DefaultSimulator.class); + private Log log = LogFactory.getLog(DefaultSimulator.class); /** * Called by isis don't modify signature */ public void simulate(SimulationContext context) throws Exception { - SimulationParameter param = context.getSimulationStorage().getParameter(); + SimulationParameter param = context.getSimulationStorage() + .getParameter(); SimulationControl control = context.getSimulationControl(); int lastYear = param.getNumberOfYear(); int lastDate = lastYear * Month.NUMBER_OF_MONTH; Date date = control.getDate(); - + ResultManager resManager = context.getResultManager(); TopiaContext db = context.getDB(); - + SiMatrix siMatrix = SiMatrix.getSiMatrix(context); GravityModel gravityModel = new GravityModel(context, siMatrix); - + PopulationMonitor populationMonitor = context.getPopulationMonitor(); MetierMonitor metierMonitor = context.getMetierMonitor(); RuleMonitor ruleMonitor = context.getRuleMonitor(); - + List<Population> allpops = siMatrix.getPopulations(date); populationMonitor.init(allpops); - + for (Population pop : allpops) { MatrixND N = param.getNumberOf(pop); N.setName(ResultName.MATRIX_ABUNDANCE); populationMonitor.setN(pop, N); } + + // reload context parameters with context.getDB() context + param.reloadContextParameters(); // // Rule initialisation @@ -113,32 +114,42 @@ control.setText("Rules initialisation:" + rules); for (Rule rule : rules) { rule.init(context); - log.info("Rule " +rule.getClass().getSimpleName()+ " initialized"); + log.info("Rule " + rule.getClass().getSimpleName() + + " initialized"); } - + // // Commit all change done un init rules methods. // context.getDB().commitTransaction(); - + // // Simulation loop // while (date.getDate() < lastDate) { + // // if user stop simulation before last year // if (control.isStopSimulationRequest()) { break; } - + control.setDate(date); control.setProgress(date.getDate()); control.setText(_("begin step " + date)); - + + // + // Reload parameters for current step + // + control.setText(_("Reloading parameters")); + // reload context parameters with db.rollbackTransaction() context + param.reloadContextParameters(); + rules = param.getRules(); + // raz des metiers interdits et des licences metierMonitor.clear(); - + if (date.getMonth().equals(Month.JANUARY)) { populationMonitor.clearCatch(); } @@ -147,17 +158,18 @@ if (siMatrix.getStrategies(date).size() > 0) { if (resManager.isEnabled(ResultName.MATRIX_NO_ACTIVITY)) { MatrixND mat = metierMonitor.getOrCreateNoActivity(date, - ResultName.MATRIX_NO_ACTIVITY, - siMatrix.getStrategies(date), - siMatrix.getMetiers(date)); + ResultName.MATRIX_NO_ACTIVITY, siMatrix + .getStrategies(date), siMatrix + .getMetiers(date)); resManager.addResult(date, mat); } } - + // // Rule condition evaluation // - control.setText("Evalute Rules conditions ("+rules.size()+" rules)"); + control.setText("Evaluate rules conditions (" + rules.size() + + " rules)"); for (Rule rule : rules) { for (Metier metier : siMatrix.getMetiers(date)) { boolean active = false; @@ -165,30 +177,34 @@ active = rule.condition(context, date, metier); } catch (Exception eee) { if (log.isWarnEnabled()) { - log.warn("Can't evaluate rule condition for: " + rule, eee); + log.warn("Can't evaluate rule condition for: " + + rule, eee); } } - ruleMonitor.setEvaluationCondition(date, rule, metier, active); + ruleMonitor.setEvaluationCondition(date, rule, metier, + active); if (active) { - log.info("Activate rule: " + rule.getClass().getSimpleName()); + log.info("Activate rule: " + + rule.getClass().getSimpleName()); resManager.addActiveRule(date, rule); } } } - + // // Rule pre action // control.setText("Do pre action Rules"); for (Rule rule : rules) { for (Metier metier : siMatrix.getMetiers(date)) { - boolean condition = ruleMonitor.getEvalutionCondition(date, rule, metier); + boolean condition = ruleMonitor.getEvalutionCondition(date, + rule, metier); if (condition) { rule.preAction(context, date, metier); } } } - + // // Keep modification's information done in rule // @@ -196,7 +212,7 @@ MatrixND metierZone = siMatrix.getMetierZone(date); resManager.addResult(date, metierZone); } - + // // Simulate one step for all pop // @@ -204,40 +220,37 @@ for (Population pop : siMatrix.getPopulations(date)) { computeMonth(context, siMatrix, date, pop); } - + // // Add some result not population dependante // - + // only if there are one or more strategy if (siMatrix.getStrategies(date).size() > 0) { control.setText("Add some results"); - if (resManager.isEnabled(ResultName.MATRIX_EFFORT_PER_STRATEGY_MET)) { - MatrixND effortPerStrategyMet = siMatrix.matrixEffortPerStrategyMet(date); + if (resManager + .isEnabled(ResultName.MATRIX_EFFORT_PER_STRATEGY_MET)) { + MatrixND effortPerStrategyMet = siMatrix + .matrixEffortPerStrategyMet(date); resManager.addResult(date, effortPerStrategyMet); } - - if (resManager.isEnabled(ResultName.MATRIX_STD_TRAVEL_EFFORT_PER_STRATEGY_MET)) { - MatrixND stdTravelEffortPerStrategyMet = siMatrix.matrixEffortPerStrategyMet(date); - resManager.addResult(date, stdTravelEffortPerStrategyMet); + + if (resManager.isEnabled(ResultName.MATRIX_EFFORT_NOMINAL_PER_STRATEGY_MET)) { + MatrixND effortNominalPerStrategyMet = siMatrix.matrixEffortNominalPerStrategyMet(date); + resManager.addResult(date, effortNominalPerStrategyMet); } - if (resManager.isEnabled(ResultName.MATRIX_EFFORT_PER_ZONE_POP)) { - MatrixND effortPerZonePop = siMatrix.matrixEffortPerStrategyMet(date); - resManager.addResult(date, effortPerZonePop); - } - - // // Add economics results // - if (!"false".equalsIgnoreCase(param.getTagValue().get("ecoResult"))) { + if (!"false".equalsIgnoreCase(param.getTagValue().get( + "ecoResult"))) { control.setText("Add economics results"); saveGravityModel(date, resManager, gravityModel); } } - + // // Add economics results // @@ -248,8 +261,6 @@ } } - - // // Rule post action // @@ -274,18 +285,25 @@ MatrixND discard = populationMonitor.getDiscard(date, pop); if (discard != null || date.getDate() == 0) { // force discard for the first month to have discard in result if (discard == null) { - discard = MatrixFactory.getInstance().create(ResultName.MATRIX_DISCARDS_PER_STR_MET_PER_ZONE_POP, - new List[]{siMatrix.getStrategies(date), - siMatrix.getMetiers(date), - pop.getPopulationGroup(), - pop.getPopulationZone()}, - new String[]{n_("Strategies"), n_("Metiers"), n_("Groups"), n_("Zones")}); + discard = MatrixFactory.getInstance().create( + ResultName.MATRIX_DISCARDS_PER_STR_MET_PER_ZONE_POP, + new List[] { siMatrix.getStrategies(date), + siMatrix.getMetiers(date), + pop.getPopulationGroup(), + pop.getPopulationZone() }, + new String[] { n_("Strategies"), + n_("Metiers"), n_("Groups"), + n_("Zones") }); } resManager.addResult(date, pop, discard); - if (resManager.isEnabled(ResultName.MATRIX_DISCARDS_WEIGHT_PER_STR_MET_PER_ZONE_POP)) { - MatrixND discardWeightPerStrategyMet = siMatrix.matrixDiscardWeightPerStrategyMetPerZonePop(pop, date, discard); - resManager.addResult(date, pop, discardWeightPerStrategyMet); + if (resManager + .isEnabled(ResultName.MATRIX_DISCARDS_WEIGHT_PER_STR_MET_PER_ZONE_POP)) { + MatrixND discardWeightPerStrategyMet = siMatrix + .matrixDiscardWeightPerStrategyMetPerZonePop( + pop, date, discard); + resManager.addResult(date, pop, + discardWeightPerStrategyMet); } } @@ -294,7 +312,8 @@ // add landing result // if (resManager.isEnabled(ResultName.MATRIX_LANDING_PER_MET)) { - MatrixND landing = MatrixFactory.getInstance().create(populationMonitor.getCatch(pop)); + MatrixND landing = MatrixFactory.getInstance().create( + populationMonitor.getCatch(pop)); if (discard != null) { landing = landing.minus(discard); } @@ -309,14 +328,14 @@ // control.setText("Rollback rules changes"); db.rollbackTransaction(); - + // // commit result // control.setText("Commit results"); TopiaContext tx = context.getDbResult(); tx.commitTransaction(); - + // // Go next step // @@ -330,45 +349,50 @@ return result; } - protected void computeMonth(SimulationContext context, SiMatrix siMatrix, Date date, Population pop) throws IsisFishException, TopiaException { + protected void computeMonth(SimulationContext context, SiMatrix siMatrix, + Date date, Population pop) throws IsisFishException, TopiaException { // to add result - ResultStorage resManager = context.getSimulationStorage().getResultStorage(); + ResultStorage resManager = context.getSimulationStorage() + .getResultStorage(); PopulationMonitor popMon = context.getPopulationMonitor(); MatrixND N = popMon.getN(pop); - + if (log.isInfoEnabled()) { - log.info("====================== begin "+ date + " - " + pop +" =========================== " + System.currentTimeMillis()); + log.info("====================== begin " + date + " - " + pop + + " =========================== " + + System.currentTimeMillis()); log.info("N: " + N); } - + // add N and biomass result now, before computation // N is reassigned during computation - resManager.addResult(date, pop, N); + resManager.addResult(date, pop, N); if (resManager.isEnabled(ResultName.MATRIX_BIOMASS)) { MatrixND biomass = siMatrix.matrixBiomass(N, pop, date); resManager.addResult(date, pop, biomass); } - + Month month = date.getMonth(); PopulationSeasonInfo info = pop.getPopulationSeasonInfo(month); - + // group change MatrixND CA = null; if (date.getDate() == 0) { - CA = MatrixFactory.getInstance().matrixId(pop.sizePopulationGroup()*pop.sizePopulationZone()); + CA = MatrixFactory.getInstance().matrixId( + pop.sizePopulationGroup() * pop.sizePopulationZone()); } else { CA = info.getGroupChangeMatrix(month); } log.debug("CA: " + CA); - + //migration - MatrixND M = info.getMigrationMatrix(month, N); + MatrixND M = info.getMigrationMatrix(month, N); log.debug("M: " + M); //emigration - MatrixND EM = info.getEmigrationMatrix(month, N); + MatrixND EM = info.getEmigrationMatrix(month, N); log.debug("EM: " + EM); //immigration MatrixND IM = info.getImmigrationMatrix(month, N).transpose(); @@ -382,9 +406,9 @@ MatrixND tmp1 = M.minus(EM); MatrixND tmp2 = tmp0.mult(tmp1); MatrixND tmp3 = tmp2.add(IM); - + log.debug("N1D after mig: " + tmp3); - + // On reconvertie en une matrice Semantique N = pop.split2D(tmp3); @@ -393,32 +417,32 @@ //Recrutement MatrixND R = info.getReproductionMatrix(month, N); log.debug("R: " + R); - + // ajout de la matrice R dans le suivi de la pop comme etant // la reproduction pour le mois courant. popMon.setReproduction(date, pop, R); - + // recrutement MatrixND recruitment = popMon.getRecruitment(date, pop); log.debug("recruitment: " + recruitment); - + // mortalite de la reproduction popMon.applyReproductionMortality(pop); - + N = N.add(recruitment); log.debug("N after recru: " + N); - if (resManager.isEnabled(ResultName.MATRIX_ABONDANCE_BEGIN_MONTH)) { - MatrixND abondanceBM = siMatrix.matrixAbondanceBeginMonth(N, pop, date); + if (resManager.isEnabled(ResultName.MATRIX_ABUNDANCE)) { + MatrixND abondanceBM = siMatrix.matrixAbondanceBeginMonth(N, pop, + date); resManager.addResult(date, pop, abondanceBM); } - if (resManager.isEnabled(ResultName.MATRIX_BIOMASS_BEGIN_MONTH)) { + if (resManager.isEnabled(ResultName.MATRIX_BIOMASS)) { MatrixND biomassBM = siMatrix.matrixBiomassBeginMonth(N, pop, date); resManager.addResult(date, pop, biomassBM); } - MatrixND abundance = N; // compute fishing matrix only if there are one or more strategy if (siMatrix.getStrategies(date).size() > 0) { @@ -429,153 +453,193 @@ if (isEffortByCell(context)) { abundance = siMatrix.matrixAbundance(N, pop, date); - catchPerStrategyMetPerZonePop = - siMatrix.matrixCatchPerStrategyMetPerZonePop(N, pop, date); + catchPerStrategyMetPerZonePop = siMatrix + .matrixCatchPerStrategyMetPerZonePop(N, pop, date); } else { // en zone - MatrixND matrixFishingMortality = siMatrix.matrixFishingMortality(date, pop); + MatrixND matrixFishingMortality = siMatrix + .matrixFishingMortality(date, pop); resManager.addResult(date, pop, matrixFishingMortality); - - abundance = siMatrix.matrixAbundance(N, pop, date, matrixFishingMortality); + abundance = siMatrix.matrixAbundance(N, pop, date, + matrixFishingMortality); + // this matrix is necessary for matrixCatchPerStrategyMet - MatrixND catchRatePerStrategyMet = siMatrix.matrixCatchRatePerStrategyMetPerZone(pop, date, matrixFishingMortality); + MatrixND catchRatePerStrategyMet = siMatrix + .matrixCatchRatePerStrategyMetPerZone(pop, date, + matrixFishingMortality); resManager.addResult(date, pop, catchRatePerStrategyMet); - catchPerStrategyMetPerZonePop = - siMatrix.matrixCatchPerStrategyMetPerZone(N, pop, date, catchRatePerStrategyMet); + catchPerStrategyMetPerZonePop = siMatrix + .matrixCatchPerStrategyMetPerZone(N, pop, date, + catchRatePerStrategyMet); } - + popMon.holdCatch(pop, catchPerStrategyMetPerZonePop); resManager.addResult(date, pop, catchPerStrategyMetPerZonePop); - - if (resManager.isEnabled(ResultName.MATRIX_CATCH_WEIGHT_PER_STRATEGY_MET_PER_ZONE_POP)) { - MatrixND catchWeightPerStrategyMet = - siMatrix.matrixCatchWeightPerStrategyMetPerZonePop(date, pop, - catchPerStrategyMetPerZonePop); + + if (resManager + .isEnabled(ResultName.MATRIX_CATCH_WEIGHT_PER_STRATEGY_MET_PER_ZONE_POP)) { + MatrixND catchWeightPerStrategyMet = siMatrix + .matrixCatchWeightPerStrategyMetPerZonePop(date, pop, + catchPerStrategyMetPerZonePop); resManager.addResult(date, pop, catchWeightPerStrategyMet); } if (isEffortByCell(context)) { - MatrixND catchPerStrategyMetPerZoneMet = - siMatrix.matrixCatchPerStrategyMetPerZoneMet(N, pop, date); + MatrixND catchPerStrategyMetPerZoneMet = siMatrix + .matrixCatchPerStrategyMetPerZoneMet(N, pop, date); resManager.addResult(date, pop, catchPerStrategyMetPerZoneMet); - if (resManager.isEnabled(ResultName.MATRIX_CATCH_WEIGHT_PER_STRATEGY_MET_PER_ZONE_MET)) { - MatrixND catchWeightPerStrategyMet = - siMatrix.matrixCatchWeightPerStrategyMetPerZoneMet(date, pop, - catchPerStrategyMetPerZoneMet); + if (resManager + .isEnabled(ResultName.MATRIX_CATCH_WEIGHT_PER_STRATEGY_MET_PER_ZONE_MET)) { + MatrixND catchWeightPerStrategyMet = siMatrix + .matrixCatchWeightPerStrategyMetPerZoneMet(date, + pop, catchPerStrategyMetPerZoneMet); resManager.addResult(date, pop, catchWeightPerStrategyMet); } } - - + } else { // no strategies // compute only if fishing mortality =0 to apply Natural Mortality abundance = siMatrix.matrixAbundanceSsF(N, pop, date); } log.debug("abundance: " + abundance); - + // Keep new N popMon.setN(pop, abundance); - log.debug("====================== end " + date + " - " + pop +" ==========================="); - + log.debug("====================== end " + date + " - " + pop + + " ==========================="); + } - + /** * @param date * @param resManager - * @throws IsisFishException - * @throws TopiaException + * @throws IsisFishException + * @throws TopiaException */ - private void saveGravityModel(Date date, ResultManager resManager, GravityModel gravityModel) throws IsisFishException, TopiaException { - if (resManager.isEnabled(ResultName.MATRIX_FISHING_TIME_PER_MONTH_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixFishingTimePerMonthPerVessel(date); + private void saveGravityModel(Date date, ResultManager resManager, + GravityModel gravityModel) throws IsisFishException, TopiaException { + if (resManager + .isEnabled(ResultName.MATRIX_FISHING_TIME_PER_MONTH_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixFishingTimePerMonthPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_FUEL_COSTS_OF_TRAVEL_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixFuelCostsOfTravelPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_FUEL_COSTS_OF_TRAVEL_PER_VESSEL)) { + MatrixND mat = gravityModel.matrixFuelCostsOfTravelPerVessel(date); resManager.addResult(date, mat); - } + } if (resManager.isEnabled(ResultName.MATRIX_COSTS_OF_FISHING_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixCostsOfFishingPerVessel(date); + MatrixND mat = gravityModel.matrixCostsOfFishingPerVessel(date); resManager.addResult(date, mat); - } + } if (resManager.isEnabled(ResultName.MATRIX_FUEL_COSTS_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixFuelCostsOfTravelPerVessel(date); + MatrixND mat = gravityModel.matrixFuelCostsOfTravelPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_REPAIR_AND_MAINTENANCE_GEAR_COSTS_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixRepairAndMaintenanceGearCostsPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_REPAIR_AND_MAINTENANCE_GEAR_COSTS_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixRepairAndMaintenanceGearCostsPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_OTHER_RUNNING_COSTS_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixOtherRunningCostsPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_OTHER_RUNNING_COSTS_PER_VESSEL)) { + MatrixND mat = gravityModel.matrixOtherRunningCostsPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_SHARED_NOT_FIXED_COSTS_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixSharedNotFixedCostsPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_SHARED_NOT_FIXED_COSTS_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixSharedNotFixedCostsPerVessel(date); resManager.addResult(date, mat); - } -// if (resManager.isEnabled(ResultName.MATRIX_SHARED_FIXED_COSTS_PER_VESSEL_PER_MET)) { -// MatrixND mat = gravityModel.matrixSharedFixedCostsPerVesselPerMet(date); -// resManager.addResult(date, mat); -// } - if (resManager.isEnabled(ResultName.MATRIX_GROSS_VALUE_OF_LANDINGS_PER_SPECIES_PER_STRATEGY_MET)) { - MatrixND mat = gravityModel.matrixGrossValueOfLandingsPerSpeciesPerStrategyMet(date); + } + // if (resManager.isEnabled(ResultName.MATRIX_SHARED_FIXED_COSTS_PER_VESSEL_PER_MET)) { + // MatrixND mat = gravityModel.matrixSharedFixedCostsPerVesselPerMet(date); + // resManager.addResult(date, mat); + // } + if (resManager + .isEnabled(ResultName.MATRIX_GROSS_VALUE_OF_LANDINGS_PER_SPECIES_PER_STRATEGY_MET)) { + MatrixND mat = gravityModel + .matrixGrossValueOfLandingsPerSpeciesPerStrategyMet(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_GROSS_VALUE_OF_LANDINGS_PER_STRATEGY_MET)) { - MatrixND mat = gravityModel.matrixGrossValueOfLandingsPerStrategyMet(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_GROSS_VALUE_OF_LANDINGS_PER_STRATEGY_MET)) { + MatrixND mat = gravityModel + .matrixGrossValueOfLandingsPerStrategyMet(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_GROSS_VALUE_OF_LANDINGS_PER_STRATEGY_MET_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixGrossValueOfLandingsPerStrategyMetPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_GROSS_VALUE_OF_LANDINGS_PER_STRATEGY_MET_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixGrossValueOfLandingsPerStrategyMetPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_NET_VALUE_OF_LANDINGS_PER_STRATEGY_MET)) { - MatrixND mat = gravityModel.matrixNetValueOfLandingsPerStrategyMet(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_NET_VALUE_OF_LANDINGS_PER_STRATEGY_MET)) { + MatrixND mat = gravityModel + .matrixNetValueOfLandingsPerStrategyMet(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_NET_VALUE_OF_LANDINGS_PER_STRATEGY_MET_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixNetValueOfLandingsPerStrategyMetPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_NET_VALUE_OF_LANDINGS_PER_STRATEGY_MET_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixNetValueOfLandingsPerStrategyMetPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_NET_RENEVUE_TO_SHARE_PER_STRATEGY_MET_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixNetRenevueToSharePerStrategyMetPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_NET_RENEVUE_TO_SHARE_PER_STRATEGY_MET_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixNetRenevueToSharePerStrategyMetPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_CREW_SHARE_PER_STRATEGY_MET_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixCrewSharePerStrategyMetPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_CREW_SHARE_PER_STRATEGY_MET_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixCrewSharePerStrategyMetPerVessel(date); resManager.addResult(date, mat); } - if (resManager.isEnabled(ResultName.MATRIX_OWNER_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_MET_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixOwnerMarginOverVariableCostsPerStrategyMetPerVessel(date); + if (resManager + .isEnabled(ResultName.MATRIX_OWNER_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_MET_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixOwnerMarginOverVariableCostsPerStrategyMetPerVessel(date); resManager.addResult(date, mat); } - if (resManager.isEnabled(ResultName.MATRIX_VESSEL_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_MET_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixVesselMarginOverVariableCostsPerStrategyMetPerVessel(date); + if (resManager + .isEnabled(ResultName.MATRIX_VESSEL_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_MET_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixVesselMarginOverVariableCostsPerStrategyMetPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_OWNER_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixOwnerMarginOverVariableCostsPerStrategyPerVessel(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_OWNER_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixOwnerMarginOverVariableCostsPerStrategyPerVessel(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_OWNER_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY)) { - MatrixND mat = gravityModel.matrixOwnerMarginOverVariableCostsPerStrategy(date); + } + if (resManager + .isEnabled(ResultName.MATRIX_OWNER_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY)) { + MatrixND mat = gravityModel + .matrixOwnerMarginOverVariableCostsPerStrategy(date); resManager.addResult(date, mat); - } - if (resManager.isEnabled(ResultName.MATRIX_VESSEL_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_PER_VESSEL)) { - MatrixND mat = gravityModel.matrixVesselMarginOverVariableCostsPerStrategyPerVessel(date); - resManager.addResult(date, mat); } - if (resManager.isEnabled(ResultName.MATRIX_VESSEL_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY)) { - MatrixND mat = gravityModel.matrixVesselMarginOverVariableCostsPerStrategy(date); + if (resManager + .isEnabled(ResultName.MATRIX_VESSEL_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY_PER_VESSEL)) { + MatrixND mat = gravityModel + .matrixVesselMarginOverVariableCostsPerStrategyPerVessel(date); resManager.addResult(date, mat); } + if (resManager + .isEnabled(ResultName.MATRIX_VESSEL_MARGIN_OVER_VARIABLE_COSTS_PER_STRATEGY)) { + MatrixND mat = gravityModel + .matrixVesselMarginOverVariableCostsPerStrategy(date); + resManager.addResult(date, mat); + } } } - -

1 0

r2827 - in isis-fish/trunk: . src/main/java/fr/ifremer/isisfish/simulator/launcher src/main/resources/templates/ssh src/test/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org Dec. 14, 2009

Dec. 14, 2009

Author: chatellier Date: 2009-12-14 14:28:56 +0000 (Mon, 14 Dec 2009) New Revision: 2827 Modified: isis-fish/trunk/changelog.txt isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java Log: Add non standalone zip management to manage AS single zip ulpoad. Modified: isis-fish/trunk/changelog.txt =================================================================== --- isis-fish/trunk/changelog.txt 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/changelog.txt 2009-12-14 14:28:56 UTC (rev 2827) @@ -1,5 +1,7 @@ isis-fish (3.3.0.0) stable; urgency=low + * Locally save simulation zip to allow simulation restart even afer isis shutdown + * In sensitivity analysis, upload only one zip to caparmor for all simulations * Add option to control number of SSH thread to use * Add split pane between queued and done simulations tables * Add free space actions on caparmor configuration UI Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-12-14 14:28:56 UTC (rev 2827) @@ -76,7 +76,18 @@ * <li>$ISIS-TMP/simulation-$id-output.txt * <li>$ISIS-TMP/simulation-$id-pbs.id * </ul> + * All $ISIS-TMP/simulation-$id-* files are deteled after result download. * + * Special case : + * <ul> + * <li>standalone zip : $ISIS-TMP/simulation-$shortid-result.zip + * (uploaded at first simulation) + * <li>standalone simulations : $ISIS-TMP/simulation-$shortid-script.seq + * (uploaded at last simulation) + * </ul> + * where {@code $shortid} is id of parent job (without increment), they are not + * deleted after result download. + * * @see JSch * * @author chatellier @@ -166,7 +177,6 @@ } return maxSimulationThread; - } /* @@ -272,8 +282,7 @@ // upload simulation on server message(control, _("isisfish.simulation.remote.message.upload")); - String simulationRemotePath = uploadSimulation(sshSession, - simulationid, simulationZip); + String simulationRemotePath = uploadSimulationIfNecessary(sshSession, simulationItem, simulationid, simulationZip); String remoteResultZip = getRemoteResultArchivePath(simulationid); @@ -640,28 +649,64 @@ } /** - * Upload simulation file to server. + * Upload simulation if necessary and always return the remote + * simulation zip path to use. * * @param session already open valid ssh session + * @param simulationItem simulation item * @param simulationid simulation id * @param simulationFile simulation file to upload * * @return remote file path or <tt>null</tt> if errors * @throws SSHException if upload fail */ - protected String uploadSimulation(Session session, String simulationid, File simulationFile) + protected String uploadSimulationIfNecessary(Session session, SimulationItem simulationItem, String simulationid, File simulationFile) throws SSHException { // first check that remote directory exists - String remotePath = getRemoteTempDirectory(); - + String remoteTemp = getRemoteTempDirectory(); + String remotePath = null; + + if (!simulationItem.isStandaloneSimulationZip()) { + // get simulation file path for each simulation... + String shortSimulationId = simulationid.substring(0, simulationid.lastIndexOf('_')); + String shortSimulationZip = "simulation-" + shortSimulationId + "-preparation.zip"; + remotePath = remoteTemp + shortSimulationZip; + + // ...but perform real upload only for frist one ! + if (simulationItem.getSimulationNumber() == 0) { + uploadSimulation(session, remoteTemp, remotePath, simulationFile); + } + } + else { + // not standalone, name always different + String simulationZip = "simulation-" + simulationid + "-preparation.zip"; + remotePath = remoteTemp + simulationZip; + // perform upload ech time + uploadSimulation(session, remoteTemp, remotePath, simulationFile); + } + + return remotePath; + } + + /** + * Perform simulation upload. + * + * Create remote temp directory if not exists. + * + * @param session already open valid ssh session + * @param remoteDirectory + * @param remoteSimulationZipPath + * + * @throws SSHException + */ + protected void uploadSimulation(Session session, String remoteDirectory, String remoteSimulationZipPath, File simulationFile) throws SSHException { + // following command work on bash and csh - String command = "test -d \"" + remotePath + "\"||mkdir -p \"" - + remotePath + "\""; + String command = "test -d \"" + remoteDirectory + "\"||mkdir -p \"" + remoteDirectory + "\""; if (log.isInfoEnabled()) { - log.info("Creating remote temp directory (if not exists) " - + remotePath); + log.info("Creating remote temp directory (if not exists) " + remoteDirectory); if (log.isDebugEnabled()) { log.debug("Executing command : " + command); } @@ -672,12 +717,7 @@ throw new SSHException(_("Command '%s' fail to execute", command)); } - // always rename simulation to "simulation-$id-preparation.zip" - remotePath += "simulation-" + simulationid + "-preparation.zip"; - - SSHUtils.scpTo(session, simulationFile, remotePath); - - return remotePath; + SSHUtils.scpTo(session, simulationFile, remoteSimulationZipPath); } /** @@ -944,22 +984,23 @@ * * Current simulation can be started later with a PBS multi job. * - * @param simulationItem simulation item (needed for additionnal info, simulation, number, idenpendant, etc...) + * @param simulationItem simulation item (needed for additionnal info, simulation, number, indenpendant, etc...) * @param simulationid simulation id - * @param simulationRemotePath simulation preparation (input) zip path + * @param simulationRemoteZipPath simulation preparation (input) zip path * @param remoteResultZip simulation result (output) zip path * @param simulationPreScriptPath simulation prescript * * @throws Exception */ - protected void startSimulation(SimulationItem simulationItem, String simulationid, String simulationRemotePath, String remoteResultZip, String simulationPreScriptPath) throws Exception { + protected void startSimulation(SimulationItem simulationItem, String simulationid, String simulationRemoteZipPath, String remoteResultZip, String simulationPreScriptPath) throws Exception { + // standalone simulation + // no question, generate script, launch it each time if (simulationItem.isStandaloneSimulation()) { // single simulation File simulationPSBScript = getLaunchSimulationScriptFile(simulationid, - simulationRemotePath, remoteResultZip, simulationPreScriptPath, false); - String scriptRemotePath = uploadSimulationScript(sshSession, - simulationid, simulationPSBScript); + simulationRemoteZipPath, true, remoteResultZip, simulationPreScriptPath, false); + String scriptRemotePath = uploadSimulationScript(sshSession, simulationid, simulationPSBScript); // prescript uploaded, delete simulationPSBScript.delete(); @@ -967,25 +1008,24 @@ sendStartSimulationRequest(sshSession, simulationid, scriptRemotePath, -1); } else { - + // standalone, on do it for last simulation if (simulationItem.isLastSimulation()) { + String shortSimulationId = simulationid.substring(0, simulationid.lastIndexOf('_')); - String shortSimulationId = simulationid.substring(0, simulationid.lastIndexOf('_')); - if (log.isDebugEnabled()) { log.debug("Last simulation start requested, send multijob start request for " + shortSimulationId); } // multiples jobs simulation File simulationPSBScript = getLaunchSimulationScriptFile(shortSimulationId, - simulationRemotePath, remoteResultZip, simulationPreScriptPath, true); - String scriptRemotePath = uploadSimulationScript(sshSession, - simulationid, simulationPSBScript); - + simulationRemoteZipPath, simulationItem.isStandaloneSimulationZip(), remoteResultZip, simulationPreScriptPath, true); + String scriptRemotePath = uploadSimulationScript(sshSession, simulationid, simulationPSBScript); + // prescript uploaded, delete simulationPSBScript.delete(); - - sendStartSimulationRequest(sshSession, simulationid, scriptRemotePath, simulationItem.getSimulationNumber()); + + // file will be named with shortSimulationId (instead of simulationId) + sendStartSimulationRequest(sshSession, shortSimulationId, scriptRemotePath, simulationItem.getSimulationNumber()); } else { if (log.isDebugEnabled()) { @@ -993,8 +1033,6 @@ } } } - - } /** @@ -1005,6 +1043,7 @@ * * @param simuationId id de la simulation * @param simulationZip zip de la simulation + * @param standaloneZip standalone simulation zip * @param preScriptPath simulation pre script path (can be null) * @param multipleSimulationScript if {@code true} build a multijob simulation script * @@ -1013,13 +1052,13 @@ * @throws IOException if can't build script */ protected File getLaunchSimulationScriptFile(String simuationId, - String simulationZip, String simulationResultZip, String preScriptPath, boolean multipleSimulationScript) throws IOException { + String simulationZip, boolean standaloneZip, String simulationResultZip, String preScriptPath, boolean multipleSimulationScript) throws IOException { File tempScript = File.createTempFile("simulation-" + simuationId + "-script", ".seq"); tempScript.deleteOnExit(); // auto delete String fileContent = getSimulationScriptLaunchContent( - QSUB_SCRIPT_TEMPLATE, simuationId, simulationZip, simulationResultZip, preScriptPath, multipleSimulationScript); + QSUB_SCRIPT_TEMPLATE, simuationId, simulationZip, standaloneZip, simulationResultZip, preScriptPath, multipleSimulationScript); FileUtil.writeString(tempScript, fileContent); return tempScript; @@ -1033,6 +1072,7 @@ * @param templateName url du template * @param simuationId id de la simulation * @param simulationZip zip de la simulation + * @param standaloneZip standalone simulation zip * @param simulationZipResult zip resultat de la simulation * @param preScriptPath simulation pre script path (can be null) * @param multipleSimulationScript if {@code true} build a multijob simulation script @@ -1040,7 +1080,7 @@ * @throws IOException if can't get script content */ protected String getSimulationScriptLaunchContent(String templateName, - String simuationId, String simulationZip, String simulationZipResult, String preScriptPath, boolean multipleSimulationScript) + String simuationId, String simulationZip, boolean standaloneZip, String simulationZipResult, String preScriptPath, boolean multipleSimulationScript) throws IOException { String scriptContent = null; @@ -1062,6 +1102,7 @@ root.put("isistemp", getRemoteTempDirectory()); root.put("simulationid", simuationId); root.put("simulationzip", simulationZip); + root.put("simulationstandalonezip", standaloneZip); root.put("simulationresultzip", simulationZipResult); root.put("simulationprescript", remotePreScript); root.put("qsubmutiplejob", multipleSimulationScript); @@ -1089,7 +1130,7 @@ * @param session valid opened session * @param simulationId simulation id (short version for a multiple job) * @param scriptRemotePath remote script path - * @param lastSimulationNumber if {@code >=0 } start a multiple pbs job form 0 to {@code lastSimulationNumber} + * @param lastSimulationNumber if {@code >=0} start a multiple pbs job form 0 to {@code lastSimulationNumber} * * @throws SSHException if call fail */ Modified: isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl 2009-12-14 14:28:56 UTC (rev 2827) @@ -9,7 +9,11 @@ <#if qsubmutiplejob> <#-- localsimulationid is short simulation id version --> <#assign localsimulationid='${simulationid}_$PBS_ARRAY_INDEX' /> - <#assign localsimulationzip='${simulationzip?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> + <#if simulationstandalonezip> + <#assign localsimulationzip='${simulationzip?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> + <#else> + <#assign localsimulationzip='${simulationzip}' /> + </#if> <#assign localsimulationresultzip='${simulationresultzip?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> <#assign localsimulationprescript='${simulationprescript?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> <#assign localsimulationoutput='${isistemp}simulation-${simulationid}_$PBS_ARRAY_INDEX-output.txt' /> Modified: isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java =================================================================== --- isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-12-14 14:28:56 UTC (rev 2827) @@ -66,7 +66,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, simulationId, - simulationZip, simulationResultZip, simulationPreScript, false); + simulationZip, true, simulationResultZip, simulationPreScript, false); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -110,7 +110,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, simulationId, - simulationZip, simulationResultZip, simulationPreScript, false); + simulationZip, true, simulationResultZip, simulationPreScript, false); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -154,7 +154,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, shortSimulationId, - simulationZip, simulationResultZip, simulationPreScript, true); + simulationZip, true, simulationResultZip, simulationPreScript, true); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -179,6 +179,9 @@ * * Le script contient plusieurs variables qui doivent être remplacées. * + * Dans ce test, le fichier zip est configuré pour être non standalone + * également. + * * @throws IOException when freemarker throws it * @throws TemplateException when freemarker throws it */ @@ -197,7 +200,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, shortSimulationId, - simulationZip, simulationResultZip, simulationPreScript, true); + simulationZip, false, simulationResultZip, simulationPreScript, true); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -205,7 +208,7 @@ // simulation parameters Assert.assertTrue("String \"" + shortSimulationId + "\" not found in template", content.indexOf(shortSimulationId + "_$PBS_ARRAY_INDEX\"") > 0); - Assert.assertTrue(content.indexOf("simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-preparation.zip") > 0); + Assert.assertTrue(content.indexOf("simulation-" + simulationId + "-preparation.zip") > 0); Assert.assertTrue(content.indexOf("/tmp/simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-result.zip") > 0); Assert.assertTrue(content.indexOf("simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-prescript.bsh") > 0); Assert.assertTrue("String " + isisTemp + "/simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-output.txt\" not found in template", content.indexOf(isisTemp + "/simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-output.txt\"") > 0);

1 0

r2826 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org Dec. 14, 2009

Dec. 14, 2009

Author: chatellier Date: 2009-12-14 14:18:13 +0000 (Mon, 14 Dec 2009) New Revision: 2826 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java Log: Set zip as not standalone for AS Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java 2009-12-14 09:58:20 UTC (rev 2825) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java 2009-12-14 14:18:13 UTC (rev 2826) @@ -473,6 +473,7 @@ SimulationItem item = new SimulationItem(realSimulationControl, subParam); item.setSimulationZip(zip); item.setStandaloneSimulation(false); // AS = always false + item.setStandaloneSimulationZip(false); // AS = same zip item.setSimulationNumber(simulationIndex); item.setLastSimulation(simulationIndex == sensitivityScenarios.getScenarios().size() - 1);

1 0

r2825 - isis-fish/trunk
by chatellier＠users.labs.libre-entreprise.org Dec. 14, 2009

Dec. 14, 2009

Author: chatellier Date: 2009-12-14 09:58:20 +0000 (Mon, 14 Dec 2009) New Revision: 2825 Modified: isis-fish/trunk/pom.xml Log: Update assembly plugin. Fix maven3 pom warnings. Modified: isis-fish/trunk/pom.xml =================================================================== --- isis-fish/trunk/pom.xml 2009-12-11 18:01:34 UTC (rev 2824) +++ isis-fish/trunk/pom.xml 2009-12-14 09:58:20 UTC (rev 2825) @@ -440,6 +440,7 @@ <plugins> <plugin> <artifactId>maven-compiler-plugin</artifactId> + <version>2.0.2</version> <configuration> <source>1.6</source> <target>1.6</target> @@ -449,6 +450,7 @@ </plugin> <plugin> <artifactId>maven-jar-plugin</artifactId> + <version>2.3</version> <configuration> <archive> <manifest> @@ -461,6 +463,7 @@ </plugin> <plugin> <artifactId>maven-surefire-plugin</artifactId> + <version>2.4.3</version> <configuration> <systemProperties> <property> @@ -584,6 +587,7 @@ <plugin> <artifactId>maven-dependency-plugin</artifactId> + <version>2.1</version> <executions> <execution> <id>copy-dependencies</id> @@ -600,7 +604,7 @@ <plugin> <artifactId>maven-assembly-plugin</artifactId> - <version>2.2-beta-4</version> + <version>2.2-beta-5</version> <configuration> <descriptors> <descriptor>src/main/assembly/bin.xml</descriptor>

1 0

r2824 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/rule
by chatellier＠users.labs.libre-entreprise.org Dec. 11, 2009

Dec. 11, 2009

Author: chatellier Date: 2009-12-11 18:01:34 +0000 (Fri, 11 Dec 2009) New Revision: 2824 Added: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/rule/package-info.java Log: Add rule package javadoc Added: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/rule/package-info.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/rule/package-info.java (rev 0) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/rule/package-info.java 2009-12-11 18:01:34 UTC (rev 2824) @@ -0,0 +1,5 @@ +/** + * Rule. + */ +package fr.ifremer.isisfish.rule; + Property changes on: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/rule/package-info.java ___________________________________________________________________ Added: svn:keywords + "Author Date Id Revision HeadURL"

1 0

r2823 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org Dec. 11, 2009

Dec. 11, 2009

Author: chatellier Date: 2009-12-11 16:35:48 +0000 (Fri, 11 Dec 2009) New Revision: 2823 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java Log: Add zip saving for each simulation. Now simulations can restart even after isis shutdown. Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java 2009-12-11 16:30:19 UTC (rev 2822) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java 2009-12-11 16:35:48 UTC (rev 2823) @@ -20,6 +20,8 @@ import static org.nuiton.i18n.I18n._; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -33,21 +35,19 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.SortedSet; import java.util.Timer; import java.util.TreeSet; import java.util.AbstractMap.SimpleEntry; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.util.FileUtil; -import fr.ifremer.isisfish.IsisFishException; -import fr.ifremer.isisfish.datastore.AnalysePlanStorage; +import fr.ifremer.isisfish.IsisFish; import fr.ifremer.isisfish.datastore.SimulationStorage; -import fr.ifremer.isisfish.simulator.AnalysePlan; import fr.ifremer.isisfish.simulator.SimulationControl; import fr.ifremer.isisfish.simulator.SimulationParameter; import fr.ifremer.isisfish.simulator.launcher.SimulationJob.PostAction; @@ -55,16 +55,22 @@ import fr.ifremer.isisfish.simulator.sensitivity.SensitivityException; /** - * Moniteur singleton pour pouvoir sauvegarder - * localement la liste des simulations démarrées et - * permettre de continuer le monitoring au relancemenent - * d'Isis. + * Moniteur singleton pour pouvoir sauvegarder localement la liste des + * simulations démarrées et permettre de continuer le monitoring au + * relancemenent d'Isis. + * + * Depuis la version 3.3, elle sauvegarde en plus les zip pour pouvoir + * relancer une simulation même si isis a été arreté entre temps. * * Cette classe n'implemente pas {@link SimulationServiceListener} * car elle ne doit pas écouter toutes les simulations, mais * seulement celle dit simple, c'est-à-dire "master plan" du plan * d'analyse. * + * Il faut bien faire attention lors du relancement des simulations a copier + * le zip a un autre endroit. Il est important d'avoir deux zip car celui + * de la sauvegarde est supprimé des que la simulation est arrete. + * * @since 3.2.0.4 * @author chatellier * @version $Revision$ @@ -74,34 +80,23 @@ */ public class SimulationMonitor extends Thread { - /** to use log facility, just put in your code: log.info(\"...\"); */ + /** Class logger. */ private static Log log = LogFactory.getLog(SimulationMonitor.class); + protected static final String PROPERTIES_FILE = "monitoring.properties"; + protected static final String PROPERTY_LAUNCHER = "simulation.launcher"; + protected static final String PROPERTY_DATE = "simulation.date"; + protected static final String PROPERTY_ZIP = "simulation.zip"; + /** Instance. */ protected static SimulationMonitor instance = new SimulationMonitor(); /** * Simulation information saving file. - * - * Saved in $HOME/.isis-simulations-3 */ - protected File monitorFile; - - /** - * Separator used in monitor file. - * - * Can't be "," - */ - protected static final String MONITORFILESEPARATOR = "/"; + protected File monitorFolder; /** - * Liste des simulations en cours, - * Le format est défini ainsi : - * id=launcher/datedelancement/onlyexport/exports/plandependant/plans - */ - protected Properties properties = new Properties(); - - /** * Map to remember when a simulation have to be checked. * * Sorted on date ASC. @@ -117,27 +112,26 @@ * Constructeur. */ protected SimulationMonitor() { - // FIXME configure this path - String monitorSaveFile = System.getProperty("user.home") - + File.separator + ".isis-simulations-3"; - monitorFile = new File(monitorSaveFile); + monitorFolder = IsisFish.config.getMonitoringDirectory(); + // init monitor check map - checkSet = new TreeSet<SimpleEntry<Date, SimulationJob>>(new Comparator<SimpleEntry<Date, SimulationJob>>(){ - @Override - public int compare(SimpleEntry<Date, SimulationJob> o1, - SimpleEntry<Date, SimulationJob> o2) { - // TODO change this, it's a set with duplicated entries !!! - int result = o1.getKey().compareTo(o2.getKey()); - // with bad luck (or windows system) date can be equals :( - if (result == 0) { - result = o1.getValue().compareTo(o2.getValue()); + checkSet = new TreeSet<SimpleEntry<Date, SimulationJob>>( + new Comparator<SimpleEntry<Date, SimulationJob>>() { + @Override + public int compare(SimpleEntry<Date, SimulationJob> o1, + SimpleEntry<Date, SimulationJob> o2) { + // TODO change this, it's a set with duplicated entries !!! + int result = o1.getKey().compareTo(o2.getKey()); + // with bad luck (or windows system) date can be equals :( + if (result == 0) { + result = o1.getValue().compareTo(o2.getValue()); + } + return result; } - return result; - } - - }); + }); + // init timer checkScheduler = new Timer(); } @@ -152,183 +146,184 @@ } /** - * Reload config and read non simulation job - * into simulation service. + * Reload config and read non simulation job into simulation service. * * @param service SimulationService */ public void reloadConfig(final SimulationService service) { + // in a thread + // don't temporize UI loading + new Thread() { + public void run() { + restartSimulationProgression(service); + } + }.start(); + } - if (monitorFile.canRead()) { - InputStream is = null; - try { - is = new FileInputStream(monitorFile); - properties.load(is); + /** + * Take all simulation in loaded file. + * And add it into simulation service. + */ + protected void restartSimulationProgression(SimulationService service) { - // in a thread - // don't temporize UI loading - new Thread() { - public void run() { - restartSimulationProgression(service); - } - }.start(); - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Can't reload monitor file", e); + File[] files = monitorFolder.listFiles(); + for (File file : files) { + // folder is composed of one folder for each simulation + if (file.isDirectory()) { + + String simulationId = file.getName(); + if (log.isInfoEnabled()) { + log.info("Restart monitoring of " + simulationId); } - } - finally { - if (is != null) { + + // FIXME date remove simulation that are older than a week ? + + File propertiesFile = new File(file, PROPERTIES_FILE); + + if (propertiesFile.isFile()) { + + InputStream propertiesFileIS = null; try { - is.close(); + propertiesFileIS = new BufferedInputStream(new FileInputStream(propertiesFile)); + Properties simulationProperties = new Properties(); + simulationProperties.load(propertiesFileIS); + + restartSimulation(service, file, simulationId, simulationProperties); } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Can't close monitor file", e); + log.error("Can't get launcher for this simulation", + e); } + } finally { + IOUtils.closeQuietly(propertiesFileIS); } + } else { + if (log.isWarnEnabled()) { + log.warn("No " + PROPERTIES_FILE + " found in " + + file.getAbsolutePath() + + "(skipping restarting)"); + } } } } - } /** - * Take all simulation in loaded file. - * And add it into simulation service. + * Restart a single simulation. + * + * @param service simulation service + * @param simulationFolder simulation information folder + * @param simulationId simulation id + * @param propertiesFile simulation properties file + * + * @throws IOException */ - protected void restartSimulationProgression(SimulationService service) { + protected void restartSimulation(SimulationService service, + File simulationFolder, String simulationId, + Properties propertiesFile) throws IOException { - for (Map.Entry<Object, Object> entry : properties.entrySet()) { - String simulationId = (String) entry.getKey(); - String simulationInfo = (String) entry.getValue(); + try { + // launcher + String simulationLauncher = propertiesFile + .getProperty(PROPERTY_LAUNCHER); + SimulatorLauncher launcher = (SimulatorLauncher) Class.forName( + simulationLauncher).newInstance(); - // 0 = launcher - // 1 = date - // 2 = export - // 3 = plan dependent - // 4 = use analyse plan - // 5 = plans list - String[] simulationInfos = simulationInfo.split(MONITORFILESEPARATOR); - String simulationLauncher = simulationInfos[0]; - String simulationDate = simulationInfos[1]; - //String simulationOnlyExport = simulationInfos[2]; - //String simulationExports = simulationInfos[3]; - //String simulationUseAnalysePlan = simulationInfos[4]; - //String simulationPlans = ""; - //if(simulationInfos.length >= 6) { - // simulationPlans = simulationInfos[5]; - //} + // zip (copy to temp in mandatory) + String zipFileName = propertiesFile.getProperty(PROPERTY_ZIP); + File zipFile = new File(simulationFolder, zipFileName); + // + File tempZipFile = File.createTempFile( + "simulation-" + simulationId, ".zip"); + FileUtils.copyFile(zipFile, tempZipFile); + tempZipFile.deleteOnExit(); - if (log.isInfoEnabled()) { - log.info("Restart monitoring of " + simulationId); - if (log.isDebugEnabled()) { - log.debug(" launcher= " + simulationLauncher + ", date = " - + simulationDate); - } + // On construit le job. + // il faut qu'il soit suffisament valide pour pourvoir démarrer + // il contient : + // - un launcher + // - un zip (pour relancement) + // Et il est relance avec un parametre "checkonly" + SimulationParameter params = new SimulationParameter(); + SimulationControl control = new SimulationControl(simulationId); + SimulationItem item = new SimulationItem(control, params); + item.setSimulationZip(tempZipFile); // tempZipFile, important ! + SimulationJob job = new SimulationJob(service, item, 0); + job.setLauncher(launcher); + + service.submitForCheckOnly(job); + } catch (ClassNotFoundException e) { + if (log.isErrorEnabled()) { + log.error("Can't found launcher for this simulation", e); } + } catch (InstantiationException e) { + if (log.isErrorEnabled()) { + log.error("Can't get launcher for this simulation", e); + } + } catch (IllegalAccessException e) { + if (log.isErrorEnabled()) { + log.error("Can't get launcher for this simulation", e); + } + } + } - // FIXME date remove simulation that are older than a week ? + /** + * Save simulation informations for started simulations. + * + * @param job started job + */ + public synchronized void simulationStart(SimulationJob job) { - // re add simulation in - try { - // launcher - SimulatorLauncher launcher = (SimulatorLauncher) Class.forName( - simulationLauncher).newInstance(); + String simulationId = job.getId(); + File simulationMonitoringFolder = new File(monitorFolder, simulationId); + File simulationPropertiesFile = new File(simulationMonitoringFolder, PROPERTIES_FILE); - SimulationParameter params = new SimulationParameter(); - // UseAnalysePlan - //params.setUseAnalysePlan("true".equalsIgnoreCase(simulationUseAnalysePlan)); - // plans - /*String[] planList = simulationPlans.split(","); - for (String name : planList) { - if (name != null && !name.isEmpty()) { - try { - AnalysePlan plan = AnalysePlanStorage.getAnalysePlan(name).getNewAnalysePlanInstance(); - params.getAnalysePlans().add(plan); - } catch (IsisFishException eee) { - if (log.isWarnEnabled()) { - log.warn("Can't find plan: " + name, eee); - } - } - } - }*/ - // OnlyExport - //params.setOnlyExport("true".equalsIgnoreCase(simulationOnlyExport)); - // exports - /*String[] exportList = simulationExports.split(","); - for (String name : exportList) { - if (name != null && !name.isEmpty()) { - params.getExportNames().add(name); - } - }*/ + if (!simulationPropertiesFile.exists()) { + Properties simulationProperties = new Properties(); - service.submitForCheckOnly(simulationId, params, launcher, 0); - } catch (ClassNotFoundException e) { + // launcher + SimulatorLauncher launcher = job.getLauncher(); + String launcherName = launcher.getClass().getName(); + simulationProperties.setProperty(PROPERTY_LAUNCHER, launcherName); + + // date + simulationProperties.setProperty(PROPERTY_DATE, launcherName); + + // zip + File zipFile = job.getItem().getSimulationZip(); + try { + // original zip must be keeped outside monitoring folder + FileUtils.copyFileToDirectory(zipFile, simulationMonitoringFolder); + simulationProperties.setProperty(PROPERTY_ZIP, zipFile.getName()); + } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Can't found launcher for this simulation", e); + log.error("Can't copy simulation zip", e); } - } catch (InstantiationException e) { + } + + // save property file + OutputStream out = null; + try { + out = new BufferedOutputStream(new FileOutputStream(simulationPropertiesFile)); + simulationProperties.store(out, "Simulation added : " + simulationId); + } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Can't get launcher for this simulation", e); + log.error("Can't save monitor file", e); } - } catch (IllegalAccessException e) { - if (log.isErrorEnabled()) { - log.error("Can't get launcher for this simulation", e); - } + } finally { + IOUtils.closeQuietly(out); } - } - } - public synchronized void simulationStart(SimulationJob job) { - - String monitorKey = job.getId(); - SimulatorLauncher launcher = job.getLauncher(); - //SimulationParameter params = job.getItem().getParameter(); - - // 0 = launcher - String monitorValue = launcher.getClass().getName(); - // 1 = date - monitorValue += MONITORFILESEPARATOR + System.currentTimeMillis(); - // 2 = onlyexport - //monitorValue += MONITORFILESEPARATOR + params.getOnlyExport(); - // 3 = exports - //monitorValue += MONITORFILESEPARATOR + collectionToString(params.getExportNames()); - // 4 = plan dependent - //monitorValue += MONITORFILESEPARATOR + params.getUseAnalysePlan(); - // 5 = plan dependent - //String planList = ""; - /*for (AnalysePlan plan : params.getAnalysePlans()) { - planList += AnalysePlanStorage.getName(plan) + ","; - } - monitorValue += MONITORFILESEPARATOR + planList;*/ - - properties.put(monitorKey, monitorValue); - OutputStream out = null; - try { - out = new FileOutputStream(monitorFile); - properties.store(new FileOutputStream(monitorFile), - "Simulation added at " + new Date()); - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Can't save monitor file", e); + if (log.isInfoEnabled()) { + log.info("Saving simulation " + simulationId + " as started"); } - } - finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Can't close monitor file", e); - } - } + } else { + if (log.isDebugEnabled()) { + log.debug("We already know information about this simulation"); + log.debug("Skip simulation information saving"); } } - if (log.isInfoEnabled()) { - log.info("Saving simulation " + monitorKey + " as started"); - } - // put interval in launcher ? Date checkDate = new Date(); // do check at start @@ -351,37 +346,20 @@ * @param job job that control the simulation */ public synchronized void simulationStop(SimulationJob job) { - String monitorKey = job.getId(); - properties.remove(monitorKey); - OutputStream out = null; + String simulationId = job.getId(); + File simulationMonitoringFolder = new File(monitorFolder, simulationId); + try { - if (properties.isEmpty()) { - monitorFile.delete(); - } else { - out = new FileOutputStream(monitorFile); - properties.store(out, "Simulation delete at " + new Date()); - - } + FileUtils.deleteDirectory(simulationMonitoringFolder); } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Can't save monitor file", e); + log.error("Can't remove simulation informations", e); } } - finally { - try { - if (out != null) { - out.close(); - } - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Can't close monitor file", e); - } - } - } if (log.isInfoEnabled()) { - log.info("Saving simulation " + monitorKey + " as stopped"); + log.info("Saving simulation " + simulationId + " as stopped"); } } @@ -398,8 +376,7 @@ // quoi qu'il arrive, le thread ne doit jamais s'arreter try { waitAndCheckProgression(); - } - catch (Exception e) { + } catch (Exception e) { if (log.isErrorEnabled()) { log.error("An exception occurs durring monitoring", e); } @@ -413,7 +390,7 @@ * is reached. */ protected void waitAndCheckProgression() { - + try { Thread.sleep(1000); } catch (InterruptedException e) { @@ -468,14 +445,15 @@ long nextJobTimeMs = date.getTime() + launcher.getCheckProgressionInterval() * 1000; Date nextJobDate = new Date(nextJobTimeMs); - checkSet.add(new SimpleEntry<Date, SimulationJob>(nextJobDate, job)); + checkSet.add(new SimpleEntry<Date, SimulationJob>( + nextJobDate, job)); } } else { //checkScheduler.schedule(this, date); if (log.isDebugEnabled()) { - log.debug("Time to check not reached, skip check, " - + date + " < " + now); + log.debug("Time to check not reached, skip check, " + date + + " < " + now); } } } @@ -510,13 +488,14 @@ // WARNING this condition is VERY important // and set by end of // fr.ifremer.isisfish.simulator.launcher.InProcessSimulatorLauncher#localSimulateSameThread(SimulationControl, SimulationStorage) - + // FIXME check SimulationStorage.exists(control.getId()) condition if ((control.getProgress() > 0 - && control.getProgress() >= control.getProgressMax() && SimulationStorage.exists(control.getId()))) { + && control.getProgress() >= control.getProgressMax() && SimulationStorage + .exists(control.getId()))) { simulationEnded = true; } - + if (control.isStopSimulationRequest()) { launcher.simulationStopRequest(job); // to release one resource simulationEnded = true; @@ -545,28 +524,28 @@ SimulatorLauncher launcher) { boolean result = false; - + SimulationService service = SimulationService.getService(); SimulationControl control = job.getItem().getControl(); - + if (log.isInfoEnabled()) { log.info("Do post simulation operation for " + control.getId()); } try { - + // get storage // meme si StopSimulationRequest // force le download dans le cas de ssh SimulationStorage simulation = launcher.getSimulationStorage( service, control); - + // on ne fait de post traitement que si // la simulation n'a pas été arretée if (!control.isStopSimulationRequest()) { // post manage this storage boolean simulationAvailble = true; //exportResult(job, simulation); - + // do posts actions for (PostAction action : job.getPostActions()) { try { @@ -577,7 +556,7 @@ } } } - + // sensitivity calculator analyze result call // can't do analyze second pass if simulation // has been deleted @@ -591,103 +570,20 @@ // don't do in finally, getSimulationStorage may fail (md5 on ssh) simulationStop(job); service.fireStopEvent(job); - + result = true; - + } catch (RemoteException e) { if (log.isErrorEnabled()) { log.error("Can't get simulation results after simulation end", e); } } - + return result; } /** - * Export result, and delete simulation if "onlyExport" has - * been selected. - * - * @return <tt>true</tt> if simulation is still available (not delete by onlyExport) - * - * @deprecated there is no export anymore - */ - protected boolean exportResult(SimulationJob job, SimulationStorage simulation) { - - boolean simulationAvailble = true; - - /*SimulationParameter param = job.getItem().getParameter(); - SimulationControl control = job.getItem().getControl(); - String id = control.getId(); - - try { - // copie les exports de simulation dans le repertoire - // souhaiter par l'utilisateur - if (param.getExportNames() != null - && param.getExportNames().size() > 0) { - - // log - if (log.isDebugEnabled()) { - log.debug("Export results"); - } - - File exportDir = SimulationStorage - .getResultExportDirectory(simulation.getDirectory()); - // FIXME verifier que pour les plans on a bien tous les - // export, c-a-d que les export son fait dans un sous rep - // portant le nom de la simu - FileUtil.copyAndRenameRecursively(exportDir, new File(param - .getExportDirectory()), exportDir.getName(), id); - } else { - // log - if (log.isDebugEnabled()) { - log.debug("Not Export results : unneeded"); - } - } - } catch (Exception eee) { - if (log.isErrorEnabled()) { - log.error(_("Can't export simulation %s", id), eee); - } - } - - try { - // Si l'utilisateur souhaite seulement les exports - // on supprimer la/les simulations. - if (param.getOnlyExport()) { - // pour les plan dependant il faut le faire apres toutes - // les simulations donc pas ici - if (job.getParentJob() == null || param.getUseAnalysePlan() - && param.isIndependentPlan()) { - - // log - if (log.isDebugEnabled()) { - log.debug("Export \"onlyExport\" : deleting simulation"); - } - - simulation.delete(false); - simulationAvailble = false; - } else { - // log - if (log.isDebugEnabled()) { - log.debug("OnlyExport options true, but plan dependant : not delete simulation"); - } - } - } else { - // log - if (log.isDebugEnabled()) { - log.debug("OnlyExport options not set : not delete simulation"); - } - } - } catch (Exception eee) { - if (log.isErrorEnabled()) { - log.error(_("Can't delete simulation %s ", id), eee); - } - }*/ - - return simulationAvailble; - } - - /** * For each finished simulation, check that all sensitivity * simulation are available (in case of sensitivity simulation only). * @@ -696,37 +592,45 @@ */ protected void analyzeSensitivityResult(SimulationJob job, SimulationStorage simulation) { - + SimulationParameter params = simulation.getParameter(); - + int numberOfSimulation = params.getNumberOfSensitivitySimulation(); - + if (numberOfSimulation > 0) { // this is a sensitivity simulation String simulationId = job.getId(); - String simulationCommonPrefix = simulationId.substring(0, simulationId.lastIndexOf("_")); - int numberFinished = getNumberOfFinishedSimulation(simulationCommonPrefix, numberOfSimulation); + String simulationCommonPrefix = simulationId.substring(0, + simulationId.lastIndexOf("_")); + int numberFinished = getNumberOfFinishedSimulation( + simulationCommonPrefix, numberOfSimulation); // si on a toutes les simulation est qu'elles sont finie // on construit la liste des simulation storage // et on l'envoie au script d'analyse de sensibilite if (numberFinished == numberOfSimulation) { - SensitivityCalculator sensitivityCalculator = params.getSensitivityCalculator(); + SensitivityCalculator sensitivityCalculator = params + .getSensitivityCalculator(); if (sensitivityCalculator != null) { if (log.isDebugEnabled()) { - log.debug("Call analyzeResult on sensitivity script " + sensitivityCalculator.getClass().getSimpleName()); + log.debug("Call analyzeResult on sensitivity script " + + sensitivityCalculator.getClass() + .getSimpleName()); } try { // get full storage list - List<SimulationStorage> simulationStorageForAnalyze = - getStorageListForSecondPass(simulationCommonPrefix, numberOfSimulation); + List<SimulationStorage> simulationStorageForAnalyze = getStorageListForSecondPass( + simulationCommonPrefix, numberOfSimulation); // build master sensitivity export directory - File masterExportDirectory = new File(SimulationStorage.getSimulationDirectory(), simulationCommonPrefix); + File masterExportDirectory = new File(SimulationStorage + .getSimulationDirectory(), + simulationCommonPrefix); // directory must already exists !!! - sensitivityCalculator.analyzeResult(simulationStorageForAnalyze, masterExportDirectory); - } - catch(SensitivityException e) { + sensitivityCalculator.analyzeResult( + simulationStorageForAnalyze, + masterExportDirectory); + } catch (SensitivityException e) { if (log.isErrorEnabled()) { log.error("Can't call analyse result", e); } @@ -735,7 +639,7 @@ } } } - + /** * Parcourt le dossier de toutes les simulation commencant par le prefix donné, * et retourne le nombre de simulation terminées. @@ -745,39 +649,40 @@ * * @return le nombre de simulation réellement terminée */ - protected int getNumberOfFinishedSimulation(String asPrefixName, int numberOfSimulation) { + protected int getNumberOfFinishedSimulation(String asPrefixName, + int numberOfSimulation) { int numberOfFinishedSimulation = 0; - + // en sens inverse, il le fera carrement moins souvent - for (int simulationIndex = numberOfSimulation - 1; simulationIndex >= 0 ; --simulationIndex) { + for (int simulationIndex = numberOfSimulation - 1; simulationIndex >= 0; --simulationIndex) { String currentId = asPrefixName + "_" + simulationIndex; if (SimulationStorage.localyExists(currentId)) { - SimulationControl currentSimulationControl = new SimulationControl(currentId); - SimulationStorage.readControl(currentId, currentSimulationControl); - + SimulationControl currentSimulationControl = new SimulationControl( + currentId); + SimulationStorage.readControl(currentId, + currentSimulationControl); + // condition for simulation "end" if (currentSimulationControl.getProgress() > 0 && currentSimulationControl.getProgress() >= currentSimulationControl.getProgressMax()) { // new finished simulation found numberOfFinishedSimulation++; - } - else { + } else { if (log.isDebugEnabled()) { log.debug("Miss simulation number = " + simulationIndex); } - + // no need to continue // if only miss one, break break; } - } - else { + } else { break; } } - + return numberOfFinishedSimulation; } @@ -791,7 +696,7 @@ * @return la liste des storages */ protected List<SimulationStorage> getStorageListForSecondPass(String asPrefixName, int numberOfSimulation) { - + // simulation start at 0 List<SimulationStorage> simulationStorageForAnalyze = new LinkedList<SimulationStorage>(); // en sens inverse, il le fera carrement moins souvent @@ -802,10 +707,10 @@ // result list is keeped sorted simulationStorageForAnalyze.add(storage); } - + return simulationStorageForAnalyze; } - + /** * Convertit une collection de string, en une chaine * séparée par des ",". @@ -815,15 +720,15 @@ */ protected static String collectionToString(Collection<String> collection) { String str = ""; - + Iterator<String> it = collection.iterator(); - while(it.hasNext()) { + while (it.hasNext()) { str += it.next(); - if(it.hasNext()) { + if (it.hasNext()) { str += ","; } } - + return str; } -} +} // SimulationMonitor Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java 2009-12-11 16:30:19 UTC (rev 2822) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationService.java 2009-12-11 16:35:48 UTC (rev 2823) @@ -555,39 +555,6 @@ return scriptContent; } - /** - * Resoumet une simulation qui a deja été démarrée, mais - * on ne faisant que du control de monitoring. - * - * @param id - * @param param - * @param launcher - * @param priority - */ - public void submitForCheckOnly(String id, SimulationParameter param, - SimulatorLauncher launcher, int priority) { - // on l'ajoute tout de suite a la liste des simulations demandee - SimulationControl control = new SimulationControl(id); - SimulationItem item = new SimulationItem(control, param); - SimulationJob job = new SimulationJob(this, item, priority); - job.setLauncher(launcher); - fireStartEvent(job); - - job.setOnlyCheckControl(true); - - //if (param.getUseAnalysePlan() && param.isIndependentPlan()) { - // // c un plan d'analyse independant, on construit toute les sous simu - // Runnable task = new PrepareSimulationJob(this, job); - // subSimulationComputationExecutor.execute(task); - //} else { - // l'item est fini d'etre initialise, on peut l'ajouter a la queue - // sauf si c'etait un plan independant ou se seront les sous simu - // qui seront dans la queue - submit(job); - //} - - } - protected void submit(SimulationJob job) { SimulatorLauncher launcher = job.getLauncher(); // on ajoute a la queue qui utilise le launcher defini dans le job @@ -628,6 +595,18 @@ } /** + * Resoumet une simulation qui a deja été démarrée, mais + * on ne faisant que du control de monitoring. + * + * @param job job to submit + */ + public void submitForCheckOnly(SimulationJob job) { + job.setOnlyCheckControl(true); + fireStartEvent(job); + submit(job); + } + + /** * Restart a job. * * This method fire "start" event. @@ -637,6 +616,8 @@ public void restart(SimulationJob job) { // lorsqu'un job est resoumit, il redevient forcement standalone job.getItem().setStandaloneSimulation(true); + // case were we restart a reloaded job + job.setOnlyCheckControl(false); job.getItem().getControl().reset(); fireStartEvent(job);

1 0

r2822 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/util/ssh
by chatellier＠users.labs.libre-entreprise.org Dec. 11, 2009

Dec. 11, 2009

Author: chatellier Date: 2009-12-11 16:30:19 +0000 (Fri, 11 Dec 2009) New Revision: 2822 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/util/ssh/ProgressMonitor.java Log: Add since on version Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/util/ssh/ProgressMonitor.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/util/ssh/ProgressMonitor.java 2009-12-11 15:23:36 UTC (rev 2821) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/util/ssh/ProgressMonitor.java 2009-12-11 16:30:19 UTC (rev 2822) @@ -24,6 +24,7 @@ * @author chatellier * @version $Revision$ * + * @since 3.3.0.0 * Last update : $Date$ * By : $Author$ */

1 0

r2821 - in isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity: . visitor
by chatellier＠users.labs.libre-entreprise.org Dec. 11, 2009

Dec. 11, 2009

Author: chatellier Date: 2009-12-11 15:23:36 +0000 (Fri, 11 Dec 2009) New Revision: 2821 Added: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/visitor/package-info.java Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/package-info.java Log: Add package documentation Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/package-info.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/package-info.java 2009-12-11 13:56:56 UTC (rev 2820) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/package-info.java 2009-12-11 15:23:36 UTC (rev 2821) @@ -1,5 +1,5 @@ /** - * Factors. + * Sensitivity analysis. * * Ce package contient les données {@link java.io.Serializable} * permettant de representer des Facteur de variation à appliquer @@ -9,4 +9,3 @@ * comme R (http://www.r-project.org/) ou autre. */ package fr.ifremer.isisfish.simulator.sensitivity; - Added: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/visitor/package-info.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/visitor/package-info.java (rev 0) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/visitor/package-info.java 2009-12-11 15:23:36 UTC (rev 2821) @@ -0,0 +1,4 @@ +/** + * XML import/export visitor. + */ +package fr.ifremer.isisfish.simulator.sensitivity.visitor; Property changes on: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/sensitivity/visitor/package-info.java ___________________________________________________________________ Added: svn:keywords + "Author Date Id Revision HeadURL"

1 0

r2820 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org Dec. 11, 2009

Dec. 11, 2009

Author: chatellier Date: 2009-12-11 13:56:56 +0000 (Fri, 11 Dec 2009) New Revision: 2820 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationItem.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/package-info.java Log: Update doc Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-12-11 13:41:02 UTC (rev 2819) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-12-11 13:56:56 UTC (rev 2820) @@ -67,13 +67,15 @@ * Isis-Fish must be installed on remote server. * * Caparmor file layout ($i = plan/as increment) : - * - $ISIS-TMP/simulation-$id-preparation.zip - * - $ISIS-TMP/simulation-$id-prescript.bsh - * - $ISIS-TMP/simulation-$id-script.seq - * - $ISIS-TMP/simulation-$id-result.zip - * - $ISIS-TMP/simulation-$id-result.zip.md5 - * - $ISIS-TMP/simulation-$id-output.txt - * - $ISIS-TMP/simulation-$id-pbs.id + * <ul> + * <li>$ISIS-TMP/simulation-$id-preparation.zip + * <li>$ISIS-TMP/simulation-$id-prescript.bsh + * <li>$ISIS-TMP/simulation-$id-script.seq + * <li>$ISIS-TMP/simulation-$id-result.zip + * <li>$ISIS-TMP/simulation-$id-result.zip.md5 + * <li>$ISIS-TMP/simulation-$id-output.txt + * <li>$ISIS-TMP/simulation-$id-pbs.id + * </ul> * * @see JSch * Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationItem.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationItem.java 2009-12-11 13:41:02 UTC (rev 2819) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationItem.java 2009-12-11 13:56:56 UTC (rev 2820) @@ -38,7 +38,7 @@ * les simulations sont indépendantes et peuvent être lancées groupées * ({@code true} par defaut) * <li>simulationStandaloneZip : si le fichier zip est different pour toute - * les simulations ou s'il peut être commun à un esemble de simulation + * les simulations ou s'il peut être commun à un ensemble de simulation * <li>simulationNumber : simulation number in non standalone simulation * <li>simulationsCount : total simulation count * <li>lastSimulation : last simulation in a simulation pool Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/package-info.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/package-info.java 2009-12-11 13:41:02 UTC (rev 2819) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/package-info.java 2009-12-11 13:56:56 UTC (rev 2820) @@ -2,10 +2,7 @@ * <h1>Isis fish simulation launchers.</h1> * * <h2>To do</h2> - * <li> voir pourquoi l'interface de visu des simulations en cours ne se - * rafraichi pas - * <li> supprimer le SimulationItem et tout mettre dans SimulationJob (reflechir - * si au niveau design c'est une bonne chose + * <ul> * <li> faire une interface graphique de monitoring des Executors (nombre de * threads actif, nombre de simulations terminer, nombre de simulation * explicitement pour cet executor, afficher l'etat pause/resume/error et @@ -17,7 +14,6 @@ * simulations. * <li> ajouter dans l'interface de vision des simulations en cours, de * modifier le launcher des simulations non lancee. - * <li> tester les differents launcher * <li> trouver une solution pour que les jobs soumis pour un launcher * particulier soit bien executer par l'executor de ce launcher. (penser au * changement de priority avec resoumission si ce choix est pris, c-a-d que le @@ -31,6 +27,7 @@ * job est directement soumis au bon executor soit ajoute a la queue general. * (ajouter un bool qui permette lors de report d'erreur de resoumettre un job * avec launcher fixe par l'utilisateur sur la queue general ou non. + * </ul> * * <h2>Architecture global</h2> * <img src="doc-files/isis-simulation.png" alt="archi"/>

1 0