Author: bleny Date: 2010-07-07 11:22:04 +0200 (Wed, 07 Jul 2010) New Revision: 98 Url: http://nuiton.org/repositories/revision/diswork/98 Log: scheduler, doc, ajout fonctionnalit?\195?\169 dans le FS, front-end web Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/HttpFrontEnd.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkConfigTest.java Modified: trunk/diswork-daemon/pom.xml trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonConcurrencyTest.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/pom.xml trunk/src/site/rst/diswork-daemon/devel.rst trunk/src/site/rst/user/how_to_use.rst Modified: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/pom.xml 2010-07-07 09:22:04 UTC (rev 98) @@ -31,6 +31,10 @@ <artifactId>commons-io</artifactId> </dependency> <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> @@ -38,6 +42,19 @@ <groupId>commons-daemon</groupId> <artifactId>commons-daemon</artifactId> </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + <version>6.1.24</version> + </dependency> <!-- test --> <dependency> Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java 2010-07-07 09:22:04 UTC (rev 98) @@ -26,9 +26,11 @@ import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; +import java.util.Date; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.quartz.CronExpression; /** * This interface introduce the concept of "activity strategy" in @@ -59,7 +61,7 @@ * */ public interface ActivityStrategy { - + /** use this strategy to never run a job */ public static class NoActivity implements ActivityStrategy { @Override @@ -126,14 +128,24 @@ /** use this strategy to run a job only at fixed times of the week */ public static class ScheduledActivity implements ActivityStrategy { + protected DisworkConfig config; + + protected ScheduledActivity(DisworkConfig config) { + this.config = config; + } + @Override - public boolean canWork() { - // TODO 20100615 bleny Auto-generated method stub - throw new UnsupportedOperationException("not yet implemented"); + public boolean canWork() throws DisworkException { + Date currentDate = new Date(); + boolean result = false; + for (CronExpression pattern : config.getSchedule()) { + result = result || pattern.isSatisfiedBy(currentDate); + } + return result; } } /** return true if a job can be run */ - boolean canWork(); + boolean canWork() throws DisworkException; } \ No newline at end of file Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-07 09:22:04 UTC (rev 98) @@ -24,20 +24,28 @@ */ package org.nuiton.diswork.daemon; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import java.io.Reader; import java.lang.management.ManagementFactory; +import java.text.ParseException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.nuiton.util.ApplicationConfig; +import org.quartz.CronExpression; /** * @@ -72,6 +80,18 @@ * this is a property file</dd> * </dl> * + * For the http front-end : + * <dl> + * <dt>diswork.httpfrontend.start</dt> + * <dd>"true" and the front end will start at diswork boot. Note that + * the front-end can be started later even if this config directive is + * at "false"</dd> + * <dt>diswork.httpfrontend.port</dt> + * <dd>the port to use for HTTP server. It means that the browser while + * have to use this port to get connected to the front-end. Default + * port is 8080.</dd> + * </dl> + * * @author bleny */ public class DisworkConfig extends ApplicationConfig { @@ -105,15 +125,33 @@ // if no total_uptime saved, consider daemon has never run setDefaultOption("diswork.total_uptime", "0"); - } - public static DisworkConfig newConfig() { - DisworkConfig newConfig = new DisworkConfig(); - newConfig.setFileSystemConfig(DisworkFileSystemConfig.newKademliaDisworkConfig()); - return newConfig; + setOption("diswork.httpfrontend.start", "true"); + setOption("diswork.httpfrontend.port", "8080"); + + setFileSystemConfig(DisworkFileSystemConfig.newKademliaDisworkConfig()); } + /** + * + * @return null if no file specified + */ + public File getTokensFile() { + String path = getOption("diswork.tokens_file"); + if (path == null) { + return null; + } else { + File file = new File(path); + return file; + } + } + + public void setTokensFile(String tokensFilePath) { + setOption("diswork.tokens_file", tokensFilePath); + } + + /** * Read the tokens file if one is given in the config and merge the content * of this file into {@link #tokens} * @@ -139,9 +177,13 @@ Properties userTokens = new Properties(); userTokens.load(tokensStream); for (String token : userTokens.stringPropertyNames()) { - log.debug("adding token " + token + " → " + - userTokens.getProperty(token)); - tokens.put(token, userTokens.getProperty(token)); + String replacement = userTokens.getProperty(token); + // prevent java to replace \t by a tab in C:\temp + String escapedReplacement = StringEscapeUtils.escapeJava + (StringEscapeUtils.escapeJava(replacement)); + + log.debug("adding token " + token + " → " + escapedReplacement); + tokens.put(token, escapedReplacement); } } catch (FileNotFoundException e) { log.warn("tokens file not found, 0 tokens loaded", e); @@ -154,6 +196,8 @@ } } + + protected String applyTokensRecursively(String commandLine) { String result = commandLine; for (String token : tokens.keySet()) { @@ -173,7 +217,9 @@ initTokens(); } - tokens.put("%tmp", tempDir); + if (tempDir != null) { + tokens.put("%tmp", tempDir); + } return applyTokensRecursively(commandLine); } @@ -183,18 +229,86 @@ - public String getTempDirectory() { - return getOption("diswork.temp_directory"); + protected List<CronExpression> schedule; + + /** + * + * @return null if no path for a file have been specified + * @throws DisworkException + */ + protected List<CronExpression> getSchedule() throws DisworkException { + // lazy instanciation of schedule + if (schedule == null) { + String path = getOption("diswork.schedule_file"); + if (path != null) { + File file = new File(path); + schedule = new ArrayList<CronExpression>(); + try { + Reader reader = new FileReader(file); + BufferedReader in = new BufferedReader(reader); + String line; + while ((line = in.readLine()) != null) { + if (!line.startsWith("#") && !"".equals(line)) { + try { + schedule.add(new CronExpression(line)); + } catch (ParseException e) { + log.warn("failed to parse " + line + " : line ignored"); + } + } + } + } catch (FileNotFoundException e) { + log.error(e); + throw new DisworkException(e); + } catch (IOException e) { + log.error(e); + throw new DisworkException(e); + } + } + } + return schedule; } + + public void setSheduleFile(String path) { + setOption("diswork.schedule_file", path); + } + + + - public String getOwnerId() { + /* ** those config data are not set by the user but computed by daemon ** */ + + protected String getOwnerId() { return getOption("diswork.owner"); } - public void setOwnerId(String ownerId) { + protected void setOwnerId(String ownerId) { setOption("diswork.owner", ownerId); } - + + protected Long getTotalUptime() { + String upTime = getOption("diswork.total_uptime"); + return Long.parseLong(upTime); + } + + protected void setTotalUptime(Long upTime) { + setOption("diswork.total_uptime", upTime.toString()); + } + + protected void setFirstRunTime(Long time) { + setOption("diswork.first_run_time", time.toString()); + } + + protected Long getFirstRunTime() { + String firstRunTime = getOption("diswork.first_run_time"); + return Long.parseLong(firstRunTime); + } + + /* ** trivial applicationConfig setters and getters ** */ + + public String getTempDirectory() { + return getOption("diswork.temp_directory"); + } + public String getBootstrapIp() { return fileSystemConfig.getBootstrapIp(); } @@ -242,40 +356,20 @@ public void setActivityStrategy(String activityStrategyName) { setOption("diswork.activity_strategy", activityStrategyName); } - - public Long getTotalUptime() { - String upTime = getOption("diswork.total_uptime"); - return Long.parseLong(upTime); + + public Boolean getStartHttpFrontend() { + return getOptionAsBoolean("diswork.httpfrontend.start"); } - public void setTotalUptime(Long upTime) { - setOption("diswork.total_uptime", upTime.toString()); + public void setStartHttpFrontend(Boolean startHttpFrontend) { + setOption("diswork.httpfrontend.start", startHttpFrontend.toString()); } - public void setFirstRunTime(Long time) { - setOption("diswork.first_run_time", time.toString()); + public Integer getHttpFrontendPort() { + return getOptionAsInt("diswork.httpfrontend.port"); } - - public Long getFirstRunTime() { - String firstRunTime = getOption("diswork.first_run_time"); - return Long.parseLong(firstRunTime); - } - /** - * - * @return null if no file specified - */ - public File getTokensFile() { - String path = getOption("diswork.tokens_file"); - if (path == null) { - return null; - } else { - File file = new File(path); - return file; - } + public void setHttpFrontendPort(Integer httpFrontendPort) { + setOption("diswork.httpfrontend.port", httpFrontendPort.toString()); } - - public void setTokensFile(String tokensFilePath) { - setOption("diswork.tokens_file", tokensFilePath); - } } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-07 09:22:04 UTC (rev 98) @@ -24,7 +24,6 @@ */ package org.nuiton.diswork.daemon; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; @@ -133,7 +132,7 @@ * * @author bleny */ -public class DisworkDaemon implements Closeable { +public class DisworkDaemon { private static final Log log = LogFactory.getLog(DisworkDaemon.class); @@ -198,12 +197,14 @@ /** worker-manager will make the daemon accomplish jobs */ protected WorkersManager workers; + protected HttpFrontEnd httpFrontEnd; + public DisworkDaemon(DisworkConfig config) throws DisworkException { this.config = config; + sessionStartTime = System.currentTimeMillis(); // step by step, set all the dependencies of the daemon - // init the file-system initFileSystem(); initOwnerIdAndHomeDir(); @@ -212,7 +213,8 @@ writeHardwareInfos(); - sessionStartTime = System.currentTimeMillis(); + httpFrontEnd = new HttpFrontEnd(config, this); + } /* *** init methods, used once by the constructor */ @@ -281,7 +283,7 @@ config.setOwnerId(ownerId); - config.setFirstRunTime(System.currentTimeMillis()); + config.setFirstRunTime(sessionStartTime); // config.saveForUser(); } @@ -324,7 +326,17 @@ throw new DisworkException("can't write hardware infos", e); } } - + + /* *** methods about the web based front end *** */ + + public void startHttpFrontEnd() throws DisworkException { + httpFrontEnd.start(); + } + + public void stopHttpFrontEnd() throws DisworkException { + httpFrontEnd.stop(); + } + /* *** methods for defining some usual paths *** */ /** @@ -412,22 +424,31 @@ throw new DisworkException("unable to publish application", e); } } - + /** - * Returns a list with all jobs submitted before. - * @return + * return a list of jobs submitted before for a particular application + * @param applicationName the name of the application, can be null + * @return the submitted jobs (all if parameter was null, and only jobs + * for a particular application if parameter was set) * @throws DisworkException */ - public List<JobDescription> getAllJobs() throws DisworkException { + public List<JobDescription> getAllJobs(String applicationName) + throws DisworkException { List<JobDescription> result = new ArrayList<JobDescription>(); try { - List<String> jobs = fileSystem.readDirectory(homeDir + "/" + JOBS_DIR); - for (String jobId : jobs) { - String jsdl = IOUtils.toString(fileSystem.read( + String jobsDir = homeDir + "/" + JOBS_DIR; + if (fileSystem.exists(jobsDir)) { + List<String> jobs = fileSystem.readDirectory(jobsDir); + for (String jobId : jobs) { + String jsdl = IOUtils.toString(fileSystem.read( getPathForJob(jobId) + "/" + JSDL_PATH)); - JobDescription jobDescription = JobDescription.parseJSDL(jsdl); - jobDescription.setJobId(jobId); - result.add(jobDescription); + JobDescription jobDescription = JobDescription.parseJSDL(jsdl); + if (applicationName == null // don't filter application + || jobDescription.getApplicationName().equals(applicationName)) { + jobDescription.setJobId(jobId); + result.add(jobDescription); + } + } } } catch (IOException e) { log.error("error in file while reading home-dir", e); @@ -437,18 +458,38 @@ throw new DisworkException("file-system error", e); } return result; + } - + /** + * Returns a list with all jobs submitted before. + * @return + * @throws DisworkException + */ + public List<JobDescription> getAllJobs() throws DisworkException { + return getAllJobs(null); + } + + /** * Cancel the submission of a job * @param jobDescription * @throws DisworkException */ - public void deleteJob(JobDescription jobDescription) { - // TODO 20100618 bleny stub - throw new UnsupportedOperationException(); + public void deleteJob(JobDescription jobDescription) throws DisworkException { + try { + String jobPath = getPathForJob(jobDescription); + if (fileSystem.exists(jobPath)) { + log.info("cancelling job " + jobDescription); + fileSystem.deleteRecursively(jobPath); + } else { + log.warn("can't cancel unknown job " + jobDescription); + } + } catch (DisworkFileSystemException e) { + log.info("error while deleting a job", e); + throw new DisworkException("error while deleting a job", e); + } } - + public void submitJob(JobDescription jobDescription) throws DisworkException { if (jobDescription.getInputData().size() + jobDescription.getInputUrls().size() @@ -531,7 +572,7 @@ throw new DisworkException(e); } } - + protected boolean checkLogContains(JobDescription job, String pattern) throws DisworkException { try { @@ -542,23 +583,27 @@ log.info("unable to read log ", e); throw new DisworkException("unable to read log ", e); } catch (DisworkFileSystemException e) { - log.error("unable to read log file" + job, e); - throw new DisworkException("unable to read log file" + job, e); + log.error("unable to read log file " + job, e); + throw new DisworkException("unable to read log file " + job, e); } } - + + public boolean isStarted(JobDescription job) throws DisworkException { + return checkLogContains(job, "STARTED"); + } + public boolean isFinished(JobDescription job) throws DisworkException { return checkLogContains(job, "FINISHED"); } - + public boolean isSuccessful(JobDescription job) throws DisworkException { return checkLogContains(job, "DONE"); } - + public boolean isFailed(JobDescription job) throws DisworkException { return isFinished(job) && !isSuccessful(job); } - + /** * Must not be called until {@link #isFinished(JobDescription)} returns * true for this job. @@ -589,22 +634,45 @@ } + + /** + * + */ + public void leave() throws DisworkException { + try { + fileSystem.deleteRecursively(homeDir); + } catch (DisworkFileSystemException e) { + log.error("error while leaving the system", e); + throw new DisworkException("error while leaving the system", e); + } + } + /** close the daemon (stop all workers) * update statistics and delete all temporary data */ - @Override - public void close() throws IOException { + public void close() throws DisworkException { if (workers != null) { workers.stop(); } + try { + fileSystem.delete(homeDir + "/" + HARDINFO_PATH); + } catch (DisworkFileSystemException e) { + log.warn(e); + } + // updating total uptime statistic - Long totalUptime = getTotalUptime(); + Long totalUptime = getTotalUptime(); log.info("saving total uptime: " + totalUptime); config.setTotalUptime(totalUptime); //config.saveForUser(); - fileSystem.close(); + try { + fileSystem.close(); + } catch (DisworkFileSystemException e) { + log.error("unable to close file system", e); + throw new DisworkException(e); + } FileUtil.deleteRecursively(config.getTempDirectory()); } @@ -624,20 +692,93 @@ * this method return 0.5. * @return */ - public Double getUptimeRatio() { + protected Double getUptimeRatio() { Double uptimeRatio = (double) getTotalUptime().longValue() / ((double) System.currentTimeMillis() - (double) config.getFirstRunTime().longValue()); return uptimeRatio; } + + public Map<String, String> getLocalStats() throws DisworkException { + Map<String, String> result = new HashMap<String, String>(); + result.put("total_uptime", getTotalUptime().toString()); + result.put("uptime_ratio", getUptimeRatio().toString()); + + // TODO 20100706 bleny compute number of jobs done, number of jobs submitted, ratio, + result.put("jobs_done", "?"); + result.put("jobs_submitted", "?"); + result.put("jobs_ratio", "?"); + + // TODO 20100706 bleny compute score + result.put("score", "0"); + return result; + } + + /** a file that contains global stats about diswork */ + protected static final String GLOBAL_STATS_PATH = "/proc/globalstats"; + protected Map<String, String> updateGlobalStats() throws DisworkException { + try { + log.info("global stats file doesn't exists, creating one"); + + Long currentTime = System.currentTimeMillis(); + Map<String, Long> stats = new HashMap<String, Long>(); + Long availableProcessors = 0L; + Map<String, String> result = new HashMap<String, String>(); + + List<String> homeDirs = fileSystem.readDirectory(HOME); + for (String homeDir : homeDirs) { + String hardInfoPath = HOME + "/" + homeDir + "/" + HARDINFO_PATH; + if (fileSystem.exists(hardInfoPath)) { + String hardInfoContent = IOUtils.toString( + fileSystem.read(hardInfoPath)); + String[] infos = hardInfoContent.split("\n"); + + // first line, reading the OS name + if (!stats.containsKey(infos[0])) { + stats.put(infos[0], 0L); + } + stats.put(infos[0], stats.get(infos[0]) + 1); + + // second line, reading the architecture + if (!stats.containsKey(infos[1])) { + stats.put(infos[1], 0L); + } + stats.put(infos[1], stats.get(infos[1]) + 1); + + // third line, reading the number of processors + availableProcessors += Integer.parseInt(infos[2]); + } + } + stats.put("available_processors", availableProcessors); + stats.put("date", currentTime); + + // write the result + String statsFileContent = ""; + for (String key : stats.keySet()) { + result.put(key, stats.get(key).toString()); + statsFileContent += key + "\t" + stats.get(key) + "\n"; + } + + log.debug("writing stats file " + statsFileContent); + fileSystem.write(GLOBAL_STATS_PATH, IOUtils.toInputStream(statsFileContent)); + + return result; + } catch (DisworkFileSystemException e) { + log.error("file system error ", e); + throw new DisworkException("file system error ", e); + } catch (IOException e) { + log.error("can't write hardware infos ", e); + throw new DisworkException("can't read hardware infos ", e); + } + } + /** get infos on hardware available on the global Diswork system * * @return * @throws DisworkException */ public Map<String, String> getGlobalStats() throws DisworkException { - final String globalStatsPath = "/proc/globalstats"; final int timeBeforeGlobalStatsAreObsolete = 1 * 60 * 60 * 1000; // in this file, key and values are split with \t and entries are split @@ -645,11 +786,10 @@ // it and date it. If file is obsolete read it and delete it try { - Long currentTime = System.currentTimeMillis(); Map<String, String> result = new HashMap<String, String>(); - if (fileSystem.exists(globalStatsPath)) { - String globalStats = IOUtils.toString(fileSystem.read(globalStatsPath)); + if (fileSystem.exists(GLOBAL_STATS_PATH)) { + String globalStats = IOUtils.toString(fileSystem.read(GLOBAL_STATS_PATH)); log.debug("global stats file found, reading " + globalStats); String[] lines = globalStats.split("\n"); for (String line : lines) { @@ -658,49 +798,14 @@ } // delete the file if it's too old + Long currentTime = System.currentTimeMillis(); Long statsTime = Long.parseLong(result.get("date")); if (currentTime - statsTime > timeBeforeGlobalStatsAreObsolete) { log.info("deleting global stats file"); - fileSystem.delete(globalStatsPath); + fileSystem.delete(GLOBAL_STATS_PATH); } } else { - log.info("global stats file doesn't exists, creating one"); - Map<String, Long> stats = new HashMap<String, Long>(); - Long availableProcessors = 0L; - - List<String> homeDirs = fileSystem.readDirectory(HOME); - for (String homeDir : homeDirs) { - String hardinfo = IOUtils.toString(fileSystem.read( - HOME + "/" + homeDir + "/" + HARDINFO_PATH)); - String[] infos = hardinfo.split("\n"); - - // reading the OS name - if (!stats.containsKey(infos[0])) { - stats.put(infos[0], 0L); - } - stats.put(infos[0], stats.get(infos[0]) + 1); - - // reading the architecture - if (!stats.containsKey(infos[1])) { - stats.put(infos[1], 0L); - } - stats.put(infos[1], stats.get(infos[1]) + 1); - - // reading the number of processors - availableProcessors += Integer.parseInt(infos[2]); - } - stats.put("available_processors", availableProcessors); - stats.put("date", currentTime); - - // write the result - String statsFileContent = ""; - for (String key : stats.keySet()) { - result.put(key, stats.get(key).toString()); - statsFileContent += key + "\t" + stats.get(key) + "\n"; - } - - log.debug("writing stats file " + statsFileContent); - fileSystem.write(globalStatsPath, IOUtils.toInputStream(statsFileContent)); + result = updateGlobalStats(); } return result; Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java 2010-07-07 09:22:04 UTC (rev 98) @@ -30,7 +30,7 @@ */ public class DisworkException extends Exception { - private static final long serialVersionUID = -6434751198109021511L; + private static final long serialVersionUID = 1L; public DisworkException(String message, Throwable cause) { super(message, cause); Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/HttpFrontEnd.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/HttpFrontEnd.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/HttpFrontEnd.java 2010-07-07 09:22:04 UTC (rev 98) @@ -0,0 +1,190 @@ +package org.nuiton.diswork.daemon; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.servlet.Context; +import org.mortbay.jetty.servlet.ServletHolder; + +public class HttpFrontEnd { + + private static final Log log = LogFactory.getLog(HttpFrontEnd.class); + + protected DisworkConfig config; + + protected Server server; + + protected DisworkDaemon daemon; + + public HttpFrontEnd(DisworkConfig config, DisworkDaemon daemon) + throws DisworkException { + this.config = config; + this.daemon = daemon; + + if (config.getStartHttpFrontend()) { + start(); + } + } + + protected void initServer() { + log.info("web server use port " + config.getHttpFrontendPort()); + server = new Server(config.getHttpFrontendPort()); + Context root = new Context(server, "/", Context.NO_SESSIONS); + root.addServlet(new ServletHolder(new MainServlet()), "/"); + } + + public void start() throws DisworkException { + if (server == null) { + initServer(); + } + log.info("starting web front-end"); + try { + server.start(); + } catch (Exception e) { + log.error("error while booting http server", e); + throw new DisworkException("error while booting http server", e); + } + } + + public void stop() throws DisworkException { + log.info("stopping web front-end"); + try { + if (server != null) { + server.stop(); + } // else server has never been started + } catch (Exception e) { + log.error("error while stopping server", e); + throw new DisworkException("error while stopping server", e); + } + } + + public class MainServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + + log.info("page request"); + + String pageContent = "<html>\n" + + "<h1>Diswork Node</h1>" + "\n\n" + + "<h2>Submitted jobs</h2>" + "\n\n" + + "<table>" + "\n" + + " <tr>" + "\n" + + " <th>Name</th>" + "\n" + + " <th>Application</th>" + "\n" + + " <th>Status</th>" + "\n" + + " </tr>" + "\n"; + + List<JobDescription> jobs; + try { + jobs = daemon.getAllJobs(); + } catch (DisworkException e) { + log.error("error while retrieving local stats", e); + throw new ServletException("error while retrieving local stats", e); + } + + if (jobs.isEmpty()) { + pageContent += " <tr>\n" + + " <td colspan=\"3\"><em>no job submitted</em></td>\n" + + " </tr>\n"; + } else { + for (JobDescription job : jobs) { + + String status = "unknown"; + try { + if (daemon.isFinished(job)) { + if (daemon.isSuccessful(job)) { + status = "done"; + } else { + status = "failed"; + } + } else { + if (daemon.isStarted(job)) { + status = "started"; + } else { + status = "waiting"; + } + } + } catch (DisworkException e) { + log.error("unable to read job status", e); + throw new ServletException("unable to read job status", e); + } + + + pageContent += " <tr>\n" + + " <td>" + job.getJobName() + "</td>\n" + + " <td>" + job.getApplicationName() + "</td>\n" + + " <td>" + status + "</td>\n" + + " </tr>\n"; + } + } + + pageContent += "</table>\n\n"; + + pageContent += "<h2>Diswork statistics</h2>\n\n"; + Map<String, String> stats; + try { + stats = daemon.getLocalStats(); + } catch (DisworkException e) { + log.error("error while retrieving local stats", e); + throw new ServletException("error while retrieving local stats", e); + } + + pageContent += "<h3>Local statistics</h3>" + "\n" + + "<table>" + "\n" + + " <tr>" + "\n" + + " <th>Key</th>" + "\n" + + " <th>Value</th>" + "\n" + + " </tr>" + "\n"; + for (String stat : stats.keySet()) { + pageContent += " <tr>" + "\n" + + " <td>" + "\n" + + " " + stat + "\n" + + " </td>" + "\n" + + " <td>" + "\n" + + " " + stats.get(stat) + "\n" + + " </td>" + "\n" + + " </tr>" + "\n"; + } + + pageContent += "</table>" + "\n" + + "<h3>Global statistics</h3>" + "\n" + + "<table>" + "\n" + + " <tr>" + "\n" + + " <th>Key</th>" + "\n" + + " <th>Value</th>" + "\n" + + " </tr>" + "\n"; + + try { + stats = daemon.getGlobalStats(); + } catch (DisworkException e) { + log.error("error while retrieving local stats", e); + throw new ServletException("error while retrieving local stats", e); + } + for (String stat : stats.keySet()) { + pageContent += " <tr>" + "\n" + + " <td>" + "\n" + + " " + stat + "\n" + + " </td>" + "\n" + + " <td>" + "\n" + + " " + stats.get(stat) + "\n" + + " </td>" + "\n" + + " </tr>" + "\n"; + } + + pageContent += "</table>\n<html>" + "\n"; + + resp.getWriter().write(pageContent); + } + } + +} Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-07-07 09:22:04 UTC (rev 98) @@ -41,13 +41,13 @@ import java.util.List; import java.util.Map; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.diswork.fs.DisworkFileSystemException; import org.nuiton.util.FileUtil; +import org.nuiton.util.StringUtil; import org.nuiton.util.ZipUtil; /** @@ -103,6 +103,8 @@ /** the current activity strategy followed by all the workers */ protected ActivityStrategy activityStrategy; + protected File applicationCache; + /** A worker search a job and execute it. * * Jobs are found on the file-system by browsing some special directories, @@ -262,20 +264,11 @@ log.info("dependency needed for " + jobDescription + " (" + jobDescription.getApplicationName() + "-" + jobDescription.getApplicationVersion() + ")"); - String applicationPath = DisworkDaemon.getPathForDependency( + File application = getApplicationData( jobDescription.getApplicationName(), jobDescription.getApplicationVersion()); - InputStream applicationData = fileSystem.read(applicationPath); - - File application = new File(jobDir, - FilenameUtils.getName(applicationPath)); - application.createNewFile(); - log.info("will create " + application.getAbsolutePath()); - OutputStream out = new FileOutputStream(application); - log.debug("starting copy of " + applicationData.available() + " bytes"); - IOUtils.copy(applicationData, out); + // unzip application log.info("unzip application start"); - // unzip application ZipUtil.uncompress(application, jobDir); log.info("unzip application finished"); } else { @@ -305,7 +298,8 @@ String commandLine = config.parseCommandLine( jobDescription.getCommandLine(), jobDir.getAbsolutePath()); - String[] commandLineElements = commandLine.split(" "); + // String[] commandLineElements = commandLine.split(" "); + String[] commandLineElements = StringUtil.split(commandLine, " "); ProcessBuilder builder = new ProcessBuilder(commandLineElements); builder.directory(jobDir); builder.redirectErrorStream(true); @@ -353,18 +347,20 @@ for (String fileName : jobDescription.getOutput()) { log.info("staging file " + fileName); File localCopy = new File(jobDir, fileName); - // FIXME 20100616 bleny may not exists if job has fail - InputStream localCopyStream = new FileInputStream(localCopy); - String filePath = jobPath + "/" + fileName; + if (localCopy.exists()) { + InputStream localCopyStream = new FileInputStream(localCopy); - // erase before write - if (fileSystem.exists(filePath)) { - fileSystem.delete(filePath); + String filePath = jobPath + "/" + fileName; + + // erase before write + if (fileSystem.exists(filePath)) { + fileSystem.delete(filePath); + } + + fileSystem.write(filePath, localCopyStream); + localCopyStream.close(); } - - fileSystem.write(filePath, localCopyStream); - localCopyStream.close(); } // clean up the job directory @@ -477,27 +473,35 @@ String jobPath = jobLinkDir + "/" + jobLinkName; - boolean jobSuccess = runJob(jobPath); - - // move the link after the job - String newDir = null; - if (jobSuccess) { - newDir = DisworkDaemon.DONE; + if (fileSystem.exists(jobPath + "/" + DisworkDaemon.JSDL_PATH)) { + + log(jobPath, "STARTED"); + + boolean jobSuccess = runJob(jobPath); + + // move the link after the job + String newDir = null; + if (jobSuccess) { + newDir = DisworkDaemon.DONE; + } else { + newDir = FAILED_MOVE.get(jobLinkDir); + } + String newPath = newDir + "/" + DisworkDaemon.newJobLinkName(); + + log.info("moving " + jobPath + " to " + newPath); + fileSystem.move(jobPath, newPath); + + // mark the job has finished if done or failed too many + // times + if ( newDir == DisworkDaemon.DONE || + newDir == DisworkDaemon.FAILED_3) { + log.info("marking " + newPath + " as finished"); + log(newPath, "FINISHED"); + } } else { - newDir = FAILED_MOVE.get(jobLinkDir); + // job has been cancelled + fileSystem.delete(jobPath); } - String newPath = newDir + "/" + DisworkDaemon.newJobLinkName(); - - log.info("moving " + jobPath + " to " + newPath); - fileSystem.move(jobPath, newPath); - - // mark the job has finished if done or failed too many - // times - if ( newDir == DisworkDaemon.DONE || - newDir == DisworkDaemon.FAILED_3) { - log.info("marking " + newPath + " as finished"); - log(newPath, "FINISHED"); - } } } @@ -507,19 +511,25 @@ @Override public void run() { while (! shouldStop) { - if (activityStrategy.canWork()) { - try { + try { + if (activityStrategy.canWork()) { findAJobAndRunIt(); - } catch (DisworkException e) { - log.error("worker error " + e); - throw new RuntimeException("worker error " + e); - } catch (IOException e) { - log.error("worker error " + e); - throw new RuntimeException("worker error " + e); - } catch (DisworkFileSystemException e) { - log.error("worker error " + e); - throw new RuntimeException("worker error " + e); + } else { + // waiting for strategy to change state + Thread.sleep(10 * 1000); } + } catch (DisworkException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); + } catch (IOException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); + } catch (DisworkFileSystemException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); + } catch (InterruptedException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); } } } @@ -529,6 +539,9 @@ this.fileSystem = fileSystem; this.config = config; + applicationCache = new File(config.getTempDirectory(), "cache"); + applicationCache.mkdirs(); + // initialize activityStrategy according to config String initialStrategy = config.getActivityStrategy(); if ("none".equals(initialStrategy)) { @@ -549,11 +562,37 @@ log.info("will start " + config.getNumberOfWorkers() + " workers"); for (int i = 1 ; i <= config.getNumberOfWorkers() ; i++) { Worker worker = new Worker(); + worker.setName("disworker-" + i); worker.start(); workers.add(worker); } } + /** read an application from the file system and use a cache */ + protected File getApplicationData(String applicationName, + String applicationVersion) + throws DisworkFileSystemException, + IOException { + File cachedApplicationData = new File(applicationCache, + applicationName + "-" + applicationVersion + ".zip"); + if (!cachedApplicationData.exists()) { + log.debug("cache fail for " + applicationName + "-" + applicationVersion); + synchronized (cachedApplicationData) { + String applicationPath = DisworkDaemon.getPathForDependency( + applicationName, applicationVersion); + InputStream applicationData = fileSystem.read(applicationPath); + cachedApplicationData.createNewFile(); + log.info("will create " + cachedApplicationData.getAbsolutePath()); + OutputStream out = new FileOutputStream(cachedApplicationData); + log.debug("starting copy of " + applicationData.available() + " bytes"); + IOUtils.copy(applicationData, out); + } + } else { + log.debug("cache matches for " + applicationName + "-" + applicationVersion); + } + return cachedApplicationData; + } + public void stop() { stop(false); } @@ -564,6 +603,8 @@ worker.shouldStop = true; } + FileUtil.deleteRecursively(applicationCache); + if( !now ) { // waiting for them to actually have finished for (Worker worker : workers) { @@ -600,7 +641,7 @@ } public void activeScheduledActivityStrategy() { - setActivityStrategy(new ActivityStrategy.ScheduledActivity()); + setActivityStrategy(new ActivityStrategy.ScheduledActivity(config)); } } \ No newline at end of file Added: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkConfigTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkConfigTest.java (rev 0) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkConfigTest.java 2010-07-07 09:22:04 UTC (rev 98) @@ -0,0 +1,47 @@ +package org.nuiton.diswork.daemon; + +import static org.junit.Assert.assertEquals; + +import java.io.File; + +import org.junit.Test; +import org.nuiton.util.FileUtil; + +public class DisworkConfigTest { + + @Test + public void testSchedule() throws Exception { + String schedule = "# every night of the week on working day" + "\n" + + "* * 22-8 ? * MON-FRI" + "\n" + + "" + "\n" + + "# all day long during the week-end" + "\n" + + "* * * ? * SAT-SUN" + "\n" + + "" + "\n"; + File scheduleFile = FileUtil.getTempFile(schedule); + + DisworkConfig config = new DisworkConfig(); + config.setSheduleFile(scheduleFile.getAbsolutePath()); + + // schedule file contains 2 expressions + assertEquals(2, config.getSchedule().size()); + } + + @Test + public void testTokens() throws Exception { + // directories can be fake, only strings are considered + String tokens = "# setting the path to python" + "\n" + + "%python=/usr/bin/python" + "\n" + + "" + "\n" + + "# overwriting temp dir C:\temp" + "\n" + + "%tmp=C:\temp" + "\n" + + "" + "\n"; + File tokensFile = FileUtil.getTempFile(tokens); + + DisworkConfig config = new DisworkConfig(); + config.setTokensFile(tokensFile.getAbsolutePath()); + + assertEquals("/usr/bin/python C:\\temp", + config.parseCommandLine("%python %tmp", null)); + } + +} Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonConcurrencyTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonConcurrencyTest.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonConcurrencyTest.java 2010-07-07 09:22:04 UTC (rev 98) @@ -1,21 +1,12 @@ package org.nuiton.diswork.daemon; -import java.io.InputStream; -import org.junit.Before; - public class DisworkDaemonConcurrencyTest extends DisworkDaemonTest { - @Before - public void setUp() throws Exception { - DisworkConfig config = DisworkConfig.newConfig(); - port += 1; - config.setUsedPort(port); + @Override + protected void setConfigs() { + config = super.newConfig(); config.setNumberOfWorkers(32); - daemon = new DisworkDaemon(config); - InputStream application = ClassLoader.getSystemResourceAsStream("fake-app-1.0.zip"); - daemon.submitApplication("fake-app", "1.0", application); } - } Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-07-07 09:22:04 UTC (rev 98) @@ -1,46 +1,19 @@ package org.nuiton.diswork.daemon; -import java.io.InputStream; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.nuiton.diswork.fs.DisworkFileSystemConfig; + public class DisworkDaemonMultipleNodesTest extends DisworkDaemonTest { - /** another deamon - * this second deamon will be connected to {@link DisworkDaemonTest#daemon} - * and will to his jobs. - */ - protected static DisworkDaemon daemon2; - - @Before @Override - public void setUp() throws Exception { - DisworkConfig config = DisworkConfig.newConfig(); - port += 1; - config.setUsedPort(port); + protected void setConfigs() { + config = super.newConfig(); config.setActivityStrategy("none"); - config.fileSystemConfig.setBlockSize(500); - daemon = new DisworkDaemon(config); - InputStream application = ClassLoader.getSystemResourceAsStream("fake-app-1.0.zip"); - daemon.submitApplication("fake-app", "1.0", application); - - - DisworkConfig config2 = DisworkConfig.newConfig(); + config2 = super.newConfig(); config2.setBootstrapIp(DisworkFileSystemConfig.getIp()); - config2.setBootstrapPort(port); - port += 1; - config2.setUsedPort(port); + config2.setBootstrapPort(config.getUsedPort()); config2.setActivityStrategy("unlimited"); - daemon2 = new DisworkDaemon(config2); } - - @After - public void tearDown() throws Exception { - daemon2.close(); - } } Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-07 09:22:04 UTC (rev 98) @@ -15,23 +15,50 @@ public class DisworkDaemonTest { + protected DisworkConfig config; + protected DisworkConfig config2; + protected static DisworkDaemon daemon; + protected static DisworkDaemon daemon2; - protected static int port = 49999; + protected static int port = 45500; + + /** a factory method to ease the creation of configs */ + protected DisworkConfig newConfig() { + DisworkConfig config = new DisworkConfig(); + port += 1; + config.setUsedPort(port); + + // useless in tests + config.setStartHttpFrontend(false); + return config; + } + /** initialize config and config2 + * override this method permit to create different configuration to test, + * see sub-classes + */ + protected void setConfigs() { + config = newConfig(); + } + @Before public void setUp() throws Exception { - DisworkConfig config = DisworkConfig.newConfig(); - port += 1; - config.setUsedPort(port); + setConfigs(); daemon = new DisworkDaemon(config); + if (config2 != null) { + daemon2 = new DisworkDaemon(config2); + } InputStream application = ClassLoader.getSystemResourceAsStream("fake-app-1.0.zip"); - daemon.submitApplication("fake-app", "1.0", application); + daemon.submitApplication("fake-app", "1.0", application); } @After public void tearDown() throws Exception { daemon.close(); + if (daemon2 != null) { + daemon2.close(); + } } @Test @@ -162,4 +189,8 @@ assertEquals(4, stats.size()); } + @Test + public void testJobsManagement() throws Exception { + assertEquals(0, daemon.getAllJobs().size()); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-07 09:22:04 UTC (rev 98) @@ -24,7 +24,6 @@ */ package org.nuiton.diswork.fs; -import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -56,7 +55,7 @@ * lead to errors. * */ -public class DisworkFileSystem implements Closeable { +public class DisworkFileSystem { private static final Log log = LogFactory.getLog(DisworkFileSystem.class); @@ -435,7 +434,7 @@ } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); - log.info("trying to remove " + path); + log.info("trying to delete " + path); delete(parent, name); } @@ -700,7 +699,34 @@ } } + + public boolean isLink(String path) throws DisworkFileSystemException { + checkPathSyntax(path); + String entry = walk(path); + if (entry == null) { + throw new DisworkFileSystemException(Type.NO_SUCH_ENTITY, path); + } + return EntryUtil.isLink(entry); + } + public boolean isFile(String path) throws DisworkFileSystemException { + checkPathSyntax(path); + String entry = walk(path); + if (entry == null) { + throw new DisworkFileSystemException(Type.NO_SUCH_FILE, path); + } + return EntryUtil.isFile(entry); + } + + public boolean isDirectory(String path) throws DisworkFileSystemException { + checkPathSyntax(path); + String entry = walk(path); + if (entry == null) { + throw new DisworkFileSystemException(Type.NO_SUCH_FILE, path); + } + return EntryUtil.isDirectory(entry); + } + public void createDirectories(String path) throws DisworkFileSystemException { log.info("trying create directories for " + path); @@ -716,7 +742,17 @@ } } } - + + public void deleteRecursively(String path) throws DisworkFileSystemException { + if (isDirectory(path)) { + List<String> directoryContent = readDirectory(path); + for (String contentName : directoryContent) { + deleteRecursively(path + EntryUtil.PATH_SEPARATOR + contentName); + } + } + delete(path); + } + /** * return the entry of the element at the end of <code>path</code> * @param path @@ -827,8 +863,7 @@ return result; } - @Override - public void close() throws IOException { + public void close() throws DisworkFileSystemException { storage.close(); } Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-07-07 09:22:04 UTC (rev 98) @@ -25,7 +25,6 @@ package org.nuiton.diswork.fs.storage; import java.io.ByteArrayInputStream; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -46,7 +45,7 @@ * * It needs a place to store data called a {@link DisworkMap}. */ -public class Storage implements Closeable { +public class Storage { private static final Log log = LogFactory.getLog(Storage.class); @@ -401,10 +400,15 @@ } } - @Override - public void close() throws IOException { + public void close() throws DisworkFileSystemException { clean(); - map.close(); + try { + map.close(); + } catch (IOException e) { + log.error("error while closing map", e); + throw new DisworkFileSystemException(Type.NETWORK_FAILURE, + "unable to close the map", e); + } } /** Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-07-07 09:22:04 UTC (rev 98) @@ -505,4 +505,17 @@ fail(); } } + + @Test + public void testDeleteRecursively() throws Exception { + try { + fileSystem.createDirectories("/dir/subdir/subsubdir"); + fileSystem.createDirectory("/dir/subdir/other"); + fileSystem.createSymbolicLink("/dir/subdir/link", "/dir/subdir/other"); + fileSystem.deleteRecursively("/dir/subdir"); + assertEquals(0, fileSystem.readDirectory("/dir").size()); + } catch (DisworkFileSystemException e) { + fail(); + } + } } Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/pom.xml 2010-07-07 09:22:04 UTC (rev 98) @@ -49,6 +49,11 @@ <version>1.4</version> </dependency> <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.5</version> + </dependency> + <dependency> <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> <version>1.2.2</version> @@ -80,6 +85,16 @@ <artifactId>commons-daemon</artifactId> <version>1.0.1</version> </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + <version>1.8.3</version> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + <version>6.1.9</version> + </dependency> Modified: trunk/src/site/rst/diswork-daemon/devel.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/devel.rst 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/src/site/rst/diswork-daemon/devel.rst 2010-07-07 09:22:04 UTC (rev 98) @@ -10,8 +10,98 @@ Overview -------- -Diswork Daemon uses Diswork File System. +Diswork Daemon uses Diswork File System. Since it's distributed, all diswork +nodes acesses to the same data. +Jobs are stored on the FS with all necessary data. A simple directory structure +make easy for diswork nodes to find those jobs. + +When jobs are finished, results are written on the file system. + +Job specification +----------------- + +First, a job has a owner. It's an important because every different owner puts +his jobs in his own home directory. For user john_doe, jobs are stored in +/home/john_doe/jobs/ + +A job is a folder that contains + ++ a job description + ++ a log file + ++ the specific data for this job + +The job description describe the details of the jobs : command line to run, +input files, output files, standard input, standard output etc. + +Here is a sample job description, as generated by isis-fish : + +:: + + <jsdl:JobDefinition xmlns="http://www.example.org/" + xmlns:jsdl="http://schemas.ggf.org/jsdl/2005/11/jsdl" + xmlns:jsdl-posix="http://schemas.ggf.org/jsdl/2005/11/jsdl-posix" + xmlns:diswork="http://nuiton.org/projects/show/diswork" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <jsdl:JobDescription> + <jsdl:JobIdentification> + <jsdl:JobName>sim__2010-07-02-12-34</jsdl:JobName> + </jsdl:JobIdentification> + <jsdl:Application> + <jsdl:ApplicationName>isis-fish</jsdl:ApplicationName> + <jsdl:ApplicationVersion>3.3.0.4</jsdl:ApplicationVersion> + <jsdl-posix:POSIXApplication> + <jsdl-posix:Executable /> + <jsdl-posix:Argument>%java -Xmx2500M -jar isis-fish-3.3.0.4/isis-fish-3.3.0.4.jar --option launch.ui false --option perform.vcsupdate false --option perform.migration false --option perform.cron false --simulateRemotelly sim__2010-07-02-12-34 simulation-sim__2010-07-02-12-34-preparation.zip simulation-sim__2010-07-02-12-34-result.zip</jsdl-posix:Argument> + <jsdl-posix:Output>simulation-sim__2010-07-02-12-34-output.txt</jsdl-posix:Output> + </jsdl-posix:POSIXApplication> + </jsdl:Application> + <jsdl:DataStaging diswork:type="in"> + <jsdl:FileName>simulation-sim__2010-07-02-12-34-preparation.zip</jsdl:FileName> + </jsdl:DataStaging> + <jsdl:DataStaging diswork:type="out"> + <jsdl:FileName>simulation-sim__2010-07-02-12-34-result.zip</jsdl:FileName> + </jsdl:DataStaging> + <jsdl:DataStaging diswork:type="out"> + <jsdl:FileName>simulation-sim__2010-07-02-12-34-output.txt</jsdl:FileName> + </jsdl:DataStaging> + </jsdl:JobDescription> + </jsdl:JobDefinition> + + +As you can read, the job description references an application (isis-fish +3.3.0.4). It's a simple zip File containing all the data that are common +to all isis-related jobs. + +The log file is a simple text file. Each time a worker try to execute the job, +he write "DONE" or "FAILED". If this is the last time the job is tried, the worker +add "FINISHED". Thus, a typical log file can be + +:: + + DONE + FINISHED + +and another would be : + +:: + + FAILED + FAILED + FAILED + FINISHED + +In the job directory will be uploaded files needed for this specific job. +According to the above JSDL file, the job directory will content a file called +"simulation-sim__2010-07-02-12-34-preparation.zip". + + +Once the job is done, the job directory will contain the files asked as results. +In the above example, two files will be available for download to the requester : +"simulation-sim__2010-07-02-12-34-result.zip" and "simulation-sim__2010-07-02-12-34-output.txt" + Submit a job ------------ Modified: trunk/src/site/rst/user/how_to_use.rst =================================================================== --- trunk/src/site/rst/user/how_to_use.rst 2010-07-01 14:35:38 UTC (rev 97) +++ trunk/src/site/rst/user/how_to_use.rst 2010-07-07 09:22:04 UTC (rev 98) @@ -12,13 +12,13 @@ svn checkout http://svn.nuiton.org/svn/diswork/trunk/ diswork cd diswork/diswork-daemon - mvn assembly:assembly -Dmaven.test.skip -DdescriptorId=jar-with-dependencies + mvn assembly:assembly -DdescriptorId=jar-with-dependencies cd ../.. # under debian commons-daemon can be found at /usr/share/java/commons-daemon.jar - jsvc -cp /usr/share/java/commons-daemon.jar:diswork/diswork-daemon/target/diswork-daemon-0.0.1-SNAPSHOT-jar-with-dependencies.jar -pidfile ./pid -outfile ./out -errfile ./err DisworkDaemonRunner + # run as root : + jsvc -cp /usr/share/java/commons-daemon.jar:/usr/share/maven-repo/log4j/log4j/1.2.15/log4j-1.2.15.jar:diswork/diswork-daemon/target/diswork-daemon-0.0.1-SNAPSHOT-jar-with-dependencies.jar -pidfile ./pid -outfile ./out -errfile ./err org.nuiton.diswork.daemon.DisworkDaemonRunner - How to make my own application ready for being run by all diswork nodes -----------------------------------------------------------------------