Author: chatellier Date: 2009-12-14 14:28:56 +0000 (Mon, 14 Dec 2009) New Revision: 2827 Modified: isis-fish/trunk/changelog.txt isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java Log: Add non standalone zip management to manage AS single zip ulpoad. Modified: isis-fish/trunk/changelog.txt =================================================================== --- isis-fish/trunk/changelog.txt 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/changelog.txt 2009-12-14 14:28:56 UTC (rev 2827) @@ -1,5 +1,7 @@ isis-fish (3.3.0.0) stable; urgency=low + * Locally save simulation zip to allow simulation restart even afer isis shutdown + * In sensitivity analysis, upload only one zip to caparmor for all simulations * Add option to control number of SSH thread to use * Add split pane between queued and done simulations tables * Add free space actions on caparmor configuration UI Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-12-14 14:28:56 UTC (rev 2827) @@ -76,7 +76,18 @@ * <li>$ISIS-TMP/simulation-$id-output.txt * <li>$ISIS-TMP/simulation-$id-pbs.id * </ul> + * All $ISIS-TMP/simulation-$id-* files are deteled after result download. * + * Special case : + * <ul> + * <li>standalone zip : $ISIS-TMP/simulation-$shortid-result.zip + * (uploaded at first simulation) + * <li>standalone simulations : $ISIS-TMP/simulation-$shortid-script.seq + * (uploaded at last simulation) + * </ul> + * where {@code $shortid} is id of parent job (without increment), they are not + * deleted after result download. + * * @see JSch * * @author chatellier @@ -166,7 +177,6 @@ } return maxSimulationThread; - } /* @@ -272,8 +282,7 @@ // upload simulation on server message(control, _("isisfish.simulation.remote.message.upload")); - String simulationRemotePath = uploadSimulation(sshSession, - simulationid, simulationZip); + String simulationRemotePath = uploadSimulationIfNecessary(sshSession, simulationItem, simulationid, simulationZip); String remoteResultZip = getRemoteResultArchivePath(simulationid); @@ -640,28 +649,64 @@ } /** - * Upload simulation file to server. + * Upload simulation if necessary and always return the remote + * simulation zip path to use. * * @param session already open valid ssh session + * @param simulationItem simulation item * @param simulationid simulation id * @param simulationFile simulation file to upload * * @return remote file path or <tt>null</tt> if errors * @throws SSHException if upload fail */ - protected String uploadSimulation(Session session, String simulationid, File simulationFile) + protected String uploadSimulationIfNecessary(Session session, SimulationItem simulationItem, String simulationid, File simulationFile) throws SSHException { // first check that remote directory exists - String remotePath = getRemoteTempDirectory(); - + String remoteTemp = getRemoteTempDirectory(); + String remotePath = null; + + if (!simulationItem.isStandaloneSimulationZip()) { + // get simulation file path for each simulation... + String shortSimulationId = simulationid.substring(0, simulationid.lastIndexOf('_')); + String shortSimulationZip = "simulation-" + shortSimulationId + "-preparation.zip"; + remotePath = remoteTemp + shortSimulationZip; + + // ...but perform real upload only for frist one ! + if (simulationItem.getSimulationNumber() == 0) { + uploadSimulation(session, remoteTemp, remotePath, simulationFile); + } + } + else { + // not standalone, name always different + String simulationZip = "simulation-" + simulationid + "-preparation.zip"; + remotePath = remoteTemp + simulationZip; + // perform upload ech time + uploadSimulation(session, remoteTemp, remotePath, simulationFile); + } + + return remotePath; + } + + /** + * Perform simulation upload. + * + * Create remote temp directory if not exists. + * + * @param session already open valid ssh session + * @param remoteDirectory + * @param remoteSimulationZipPath + * + * @throws SSHException + */ + protected void uploadSimulation(Session session, String remoteDirectory, String remoteSimulationZipPath, File simulationFile) throws SSHException { + // following command work on bash and csh - String command = "test -d \"" + remotePath + "\"||mkdir -p \"" - + remotePath + "\""; + String command = "test -d \"" + remoteDirectory + "\"||mkdir -p \"" + remoteDirectory + "\""; if (log.isInfoEnabled()) { - log.info("Creating remote temp directory (if not exists) " - + remotePath); + log.info("Creating remote temp directory (if not exists) " + remoteDirectory); if (log.isDebugEnabled()) { log.debug("Executing command : " + command); } @@ -672,12 +717,7 @@ throw new SSHException(_("Command '%s' fail to execute", command)); } - // always rename simulation to "simulation-$id-preparation.zip" - remotePath += "simulation-" + simulationid + "-preparation.zip"; - - SSHUtils.scpTo(session, simulationFile, remotePath); - - return remotePath; + SSHUtils.scpTo(session, simulationFile, remoteSimulationZipPath); } /** @@ -944,22 +984,23 @@ * * Current simulation can be started later with a PBS multi job. * - * @param simulationItem simulation item (needed for additionnal info, simulation, number, idenpendant, etc...) + * @param simulationItem simulation item (needed for additionnal info, simulation, number, indenpendant, etc...) * @param simulationid simulation id - * @param simulationRemotePath simulation preparation (input) zip path + * @param simulationRemoteZipPath simulation preparation (input) zip path * @param remoteResultZip simulation result (output) zip path * @param simulationPreScriptPath simulation prescript * * @throws Exception */ - protected void startSimulation(SimulationItem simulationItem, String simulationid, String simulationRemotePath, String remoteResultZip, String simulationPreScriptPath) throws Exception { + protected void startSimulation(SimulationItem simulationItem, String simulationid, String simulationRemoteZipPath, String remoteResultZip, String simulationPreScriptPath) throws Exception { + // standalone simulation + // no question, generate script, launch it each time if (simulationItem.isStandaloneSimulation()) { // single simulation File simulationPSBScript = getLaunchSimulationScriptFile(simulationid, - simulationRemotePath, remoteResultZip, simulationPreScriptPath, false); - String scriptRemotePath = uploadSimulationScript(sshSession, - simulationid, simulationPSBScript); + simulationRemoteZipPath, true, remoteResultZip, simulationPreScriptPath, false); + String scriptRemotePath = uploadSimulationScript(sshSession, simulationid, simulationPSBScript); // prescript uploaded, delete simulationPSBScript.delete(); @@ -967,25 +1008,24 @@ sendStartSimulationRequest(sshSession, simulationid, scriptRemotePath, -1); } else { - + // standalone, on do it for last simulation if (simulationItem.isLastSimulation()) { + String shortSimulationId = simulationid.substring(0, simulationid.lastIndexOf('_')); - String shortSimulationId = simulationid.substring(0, simulationid.lastIndexOf('_')); - if (log.isDebugEnabled()) { log.debug("Last simulation start requested, send multijob start request for " + shortSimulationId); } // multiples jobs simulation File simulationPSBScript = getLaunchSimulationScriptFile(shortSimulationId, - simulationRemotePath, remoteResultZip, simulationPreScriptPath, true); - String scriptRemotePath = uploadSimulationScript(sshSession, - simulationid, simulationPSBScript); - + simulationRemoteZipPath, simulationItem.isStandaloneSimulationZip(), remoteResultZip, simulationPreScriptPath, true); + String scriptRemotePath = uploadSimulationScript(sshSession, simulationid, simulationPSBScript); + // prescript uploaded, delete simulationPSBScript.delete(); - - sendStartSimulationRequest(sshSession, simulationid, scriptRemotePath, simulationItem.getSimulationNumber()); + + // file will be named with shortSimulationId (instead of simulationId) + sendStartSimulationRequest(sshSession, shortSimulationId, scriptRemotePath, simulationItem.getSimulationNumber()); } else { if (log.isDebugEnabled()) { @@ -993,8 +1033,6 @@ } } } - - } /** @@ -1005,6 +1043,7 @@ * * @param simuationId id de la simulation * @param simulationZip zip de la simulation + * @param standaloneZip standalone simulation zip * @param preScriptPath simulation pre script path (can be null) * @param multipleSimulationScript if {@code true} build a multijob simulation script * @@ -1013,13 +1052,13 @@ * @throws IOException if can't build script */ protected File getLaunchSimulationScriptFile(String simuationId, - String simulationZip, String simulationResultZip, String preScriptPath, boolean multipleSimulationScript) throws IOException { + String simulationZip, boolean standaloneZip, String simulationResultZip, String preScriptPath, boolean multipleSimulationScript) throws IOException { File tempScript = File.createTempFile("simulation-" + simuationId + "-script", ".seq"); tempScript.deleteOnExit(); // auto delete String fileContent = getSimulationScriptLaunchContent( - QSUB_SCRIPT_TEMPLATE, simuationId, simulationZip, simulationResultZip, preScriptPath, multipleSimulationScript); + QSUB_SCRIPT_TEMPLATE, simuationId, simulationZip, standaloneZip, simulationResultZip, preScriptPath, multipleSimulationScript); FileUtil.writeString(tempScript, fileContent); return tempScript; @@ -1033,6 +1072,7 @@ * @param templateName url du template * @param simuationId id de la simulation * @param simulationZip zip de la simulation + * @param standaloneZip standalone simulation zip * @param simulationZipResult zip resultat de la simulation * @param preScriptPath simulation pre script path (can be null) * @param multipleSimulationScript if {@code true} build a multijob simulation script @@ -1040,7 +1080,7 @@ * @throws IOException if can't get script content */ protected String getSimulationScriptLaunchContent(String templateName, - String simuationId, String simulationZip, String simulationZipResult, String preScriptPath, boolean multipleSimulationScript) + String simuationId, String simulationZip, boolean standaloneZip, String simulationZipResult, String preScriptPath, boolean multipleSimulationScript) throws IOException { String scriptContent = null; @@ -1062,6 +1102,7 @@ root.put("isistemp", getRemoteTempDirectory()); root.put("simulationid", simuationId); root.put("simulationzip", simulationZip); + root.put("simulationstandalonezip", standaloneZip); root.put("simulationresultzip", simulationZipResult); root.put("simulationprescript", remotePreScript); root.put("qsubmutiplejob", multipleSimulationScript); @@ -1089,7 +1130,7 @@ * @param session valid opened session * @param simulationId simulation id (short version for a multiple job) * @param scriptRemotePath remote script path - * @param lastSimulationNumber if {@code >=0 } start a multiple pbs job form 0 to {@code lastSimulationNumber} + * @param lastSimulationNumber if {@code >=0} start a multiple pbs job form 0 to {@code lastSimulationNumber} * * @throws SSHException if call fail */ Modified: isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/src/main/resources/templates/ssh/qsub-script.ftl 2009-12-14 14:28:56 UTC (rev 2827) @@ -9,7 +9,11 @@ <#if qsubmutiplejob> <#-- localsimulationid is short simulation id version --> <#assign localsimulationid='${simulationid}_$PBS_ARRAY_INDEX' /> - <#assign localsimulationzip='${simulationzip?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> + <#if simulationstandalonezip> + <#assign localsimulationzip='${simulationzip?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> + <#else> + <#assign localsimulationzip='${simulationzip}' /> + </#if> <#assign localsimulationresultzip='${simulationresultzip?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> <#assign localsimulationprescript='${simulationprescript?replace("${simulationid}_[0-9]+", "${simulationid}_\\\\$PBS_ARRAY_INDEX","ri")}' /> <#assign localsimulationoutput='${isistemp}simulation-${simulationid}_$PBS_ARRAY_INDEX-output.txt' /> Modified: isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java =================================================================== --- isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-12-14 14:18:13 UTC (rev 2826) +++ isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-12-14 14:28:56 UTC (rev 2827) @@ -66,7 +66,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, simulationId, - simulationZip, simulationResultZip, simulationPreScript, false); + simulationZip, true, simulationResultZip, simulationPreScript, false); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -110,7 +110,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, simulationId, - simulationZip, simulationResultZip, simulationPreScript, false); + simulationZip, true, simulationResultZip, simulationPreScript, false); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -154,7 +154,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, shortSimulationId, - simulationZip, simulationResultZip, simulationPreScript, true); + simulationZip, true, simulationResultZip, simulationPreScript, true); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -179,6 +179,9 @@ * * Le script contient plusieurs variables qui doivent être remplacées. * + * Dans ce test, le fichier zip est configuré pour être non standalone + * également. + * * @throws IOException when freemarker throws it * @throws TemplateException when freemarker throws it */ @@ -197,7 +200,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, shortSimulationId, - simulationZip, simulationResultZip, simulationPreScript, true); + simulationZip, false, simulationResultZip, simulationPreScript, true); if (log.isDebugEnabled()) { log.debug("Script content = " + content); @@ -205,7 +208,7 @@ // simulation parameters Assert.assertTrue("String \"" + shortSimulationId + "\" not found in template", content.indexOf(shortSimulationId + "_$PBS_ARRAY_INDEX\"") > 0); - Assert.assertTrue(content.indexOf("simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-preparation.zip") > 0); + Assert.assertTrue(content.indexOf("simulation-" + simulationId + "-preparation.zip") > 0); Assert.assertTrue(content.indexOf("/tmp/simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-result.zip") > 0); Assert.assertTrue(content.indexOf("simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-prescript.bsh") > 0); Assert.assertTrue("String " + isisTemp + "/simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-output.txt\" not found in template", content.indexOf(isisTemp + "/simulation-" + shortSimulationId + "_$PBS_ARRAY_INDEX-output.txt\"") > 0);