Author: bleny Date: 2010-06-11 17:52:09 +0200 (Fri, 11 Jun 2010) New Revision: 73 Url: http://nuiton.org/repositories/revision/diswork/73 Log: impl?\195?\169mentation du d?\195?\169mon diswork ; ajout d'op?\195?\169rations dans diswork-fs Added: trunk/diswork-daemon/src/main/java/org/ trunk/diswork-daemon/src/main/java/org/nuiton/ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java trunk/diswork-daemon/src/test/java/org/ trunk/diswork-daemon/src/test/java/org/nuiton/ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-daemon/src/test/resources/ trunk/diswork-daemon/src/test/resources/fake-app-1.0.zip trunk/diswork-daemon/src/test/resources/fake-app.jar trunk/diswork-daemon/src/test/resources/input.txt trunk/diswork-daemon/src/test/resources/log4j.properties trunk/diswork-fs/src/test/resources/ trunk/diswork-fs/src/test/resources/log4j.properties Removed: trunk/diswork-fs/src/main/resources/log4j.properties Modified: trunk/diswork-daemon/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java Modified: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-daemon/pom.xml 2010-06-11 15:52:09 UTC (rev 73) @@ -31,6 +31,10 @@ <artifactId>commons-io</artifactId> </dependency> <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,94 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.util.ApplicationConfig; + +public class DisworkConfig extends ApplicationConfig { + + protected DisworkFileSystemConfig fileSystemConfig; + + public DisworkConfig() { + setConfigFileName("diswork.config"); + } + + public String getTempDirectory() { + return System.getProperty("java.io.tmpdir", + System.getProperty("user.dir", + ".")) + "/diswork"; + } + + public static DisworkConfig newConfig() { + DisworkConfig newConfig = new DisworkConfig(); + newConfig.setFileSystemConfig(DisworkFileSystemConfig.newKademliaDisworkConfig()); + return newConfig; + } + + public String getOwnerId() { + return getOption("diswork.owner"); + } + + public void setOwnerId(String ownerId) { + setOption("diswork.owner", ownerId); + } + + + + + + public String getBootstrapIp() { + return fileSystemConfig.getBootstrapIp(); + } + + public Integer getBootstrapPort() { + return fileSystemConfig.getBootstrapPort(); + } + + public Integer getUsedPort() { + return fileSystemConfig.getUsedPort(); + } + + public void setBootstrapIp(String ip) { + fileSystemConfig.setBootstrapIp(ip); + } + + public void setBootstrapPort(Integer port) { + fileSystemConfig.setBootstrapPort(port); + } + + public void setUsedPort(Integer port) { + fileSystemConfig.setUsedPort(port); + } + + public DisworkFileSystemConfig getFileSystemConfig() { + return fileSystemConfig; + } + + public void setFileSystemConfig(DisworkFileSystemConfig fileSystemConfig) { + this.fileSystemConfig = fileSystemConfig; + } + +} Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,372 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.UnknownHostException; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +/** + * + * As far as possible, the use of the file system follow the UNIX Filesystem + * Hierarchy Standard + * + * @see http://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard + * + * @author bleny + */ +public class DisworkDaemon implements Closeable { + + private static final Log log = LogFactory.getLog(DisworkDaemon.class); + + protected DisworkFileSystem fileSystem; + + protected DisworkConfig config; + + /** owner id for this node */ + protected String ownerId; + + /** path to owned directory on fileSystem */ + protected String homeDir; + + /** contains applications */ + protected static final String BIN = "/bin"; + + /** new jobs */ + protected static final String TODO = "/var/jobs/todo"; + + /** jobs taken that was never tried */ + protected static final String TODO_RUNNING = "/var/jobs/todo_running"; + + /** jobs failed once */ + protected static final String FAILED_1 = "/var/jobs/failed_1"; + + /** jobs taken that was failed once */ + protected static final String FAILED_1_RUNNING = "/var/jobs/failed_1_running"; + + /** jobs failed two times */ + protected static final String FAILED_2 = "/var/jobs/failed_2"; + + /** jobs taken that was failed two times */ + protected static final String FAILED_2_RUNNING = "/var/jobs/failed_2_running"; + + /** jobs failed 3 times */ + protected static final String FAILED_3 = "/var/jobs/failed_3"; + + /** jobs done */ + protected static final String DONE = "/var/jobs/done"; + + /** a place where are all user-directories */ + protected static final String HOME = "/home"; + + /** worker-manager will make the daemon accomplish jobs */ + protected WorkersManager workers; + + /** in a job directory, the place where the JSDL must be placed */ + protected static final String JSDL_PATH = ".diswork/job.jsdl"; + + /** in a job directory, the place where the log must be placed */ + protected static final String LOG_PATH = ".diswork/job.log"; + + public DisworkDaemon(DisworkConfig config) throws DisworkException { + this.config = config; + + // init fileSystem with all needed directories + try { + DisworkFileSystemConfig fileSystemConfig = + config.getFileSystemConfig(); + + fileSystem = new DisworkFileSystem(fileSystemConfig); + initFileSystem(); + } catch (UnknownHostException e) { + log.error("bootstrap failed", e); + throw new DisworkException("bootstrap failed", e); + } catch (IOException e) { + log.error("booting diswork file system failed", e); + throw new DisworkException("booting diswork file system failed", e); + } + + + ownerId = config.getOwnerId(); // get job owner id from config + + if (ownerId == null) { + log.info("can't find owner id, generating a new one"); + // generate a new one by cheking if home dir exists + ownerId = System.getProperty("user.name", "anonymous"); + // check home dir do not exists + config.setOwnerId(ownerId); + + // config.saveForUser(); + } + + log.info("owner id is " + ownerId); + + try { + homeDir = HOME + "/" + ownerId; + if (!fileSystem.exists(homeDir)) { + fileSystem.createDirectory(homeDir); + } + } catch (ConcurrentModificationException e) { + log.info("can't create home dir", e); + throw new DisworkException("can't create home dir", e); + } catch (IOException e) { + log.info("can't create home dir", e); + throw new DisworkException("can't create home dir", e); + } + + // check if config implies to run a worker + workers = new WorkersManager(fileSystem, config); + workers.start(); + } + + protected void initFileSystem() throws ConcurrentModificationException, + IOException { + String[] directories = { TODO, TODO_RUNNING, FAILED_1, FAILED_1_RUNNING, + FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE, HOME, BIN }; + // if HOME exists, we suppose all others exists + if (! fileSystem.exists(HOME)) { + for (String directory : directories) { + if (! fileSystem.exists(directory)) { + fileSystem.createDirectories(directory); + log.info("created " + directory); + } + } + } + } + + /** + * given a name and a version for an application, returns the path where + * application data can be found on diswork file system. + * @param applicationName + * @param applicationVersion + * @return a path + */ + protected static String getPathForDependency(String applicationName, + String applicationVersion) { + + String result = "/bin/" + applicationName // application directory + + "/" + // application file name + + applicationName + "-" + applicationVersion + ".zip"; + return result; + } + + /** + * every-time a link to a job is created or modified in the job, his name + * has to be generated by this method + * @return the name to use for a link + */ + protected static String newJobLinkName() { + return ((Long) System.currentTimeMillis()).toString(); + } + + public void submitApplication(String applicationName, + String applicationVersion, + InputStream applicationData) throws DisworkException { + + // the place where dependency will be stored + String path = getPathForDependency(applicationName, applicationVersion); + + String applicationDirectory = FilenameUtils.getFullPathNoEndSeparator(path); + + try { + if (!fileSystem.exists(applicationDirectory)) { + fileSystem.createDirectory(applicationDirectory); + } + + if (!fileSystem.exists(path)) { + fileSystem.write(path, applicationData); + } + } catch (ConcurrentModificationException e) { + log.info("unable to write", e); + throw new DisworkException("unable to write", e); + } catch (IOException e) { + log.info("unable to write", e); + throw new DisworkException("unable to write", e); + } + } + + /** + * Given a job description, returns the place on disworkFS where all data + * for this jobs should be stored + * @param jobDescription + * @return a path + */ + protected String getJobPath(JobDescription jobDescription) { + return getJobPath(jobDescription.getJobId()); + } + + /** + * Given a job description, returns the place on disworkFS where all data + * for this jobs should be stored + * @param jobDescription + * @return a path + */ + protected String getJobPath(String jobId) { + // all jobs are stored in home dir + return homeDir + "/" + jobId; + } + + /** + * + */ + public JobDescription newJob() throws IOException { + Random random = new Random(); + + boolean alreadyExists = true; + String newJobIntendifier = null; + while (alreadyExists) { + Integer randomInteger = random.nextInt(); + newJobIntendifier = "job_" + randomInteger.toString(); + alreadyExists = fileSystem.exists(getJobPath(newJobIntendifier)); + } + + // create both job path and sub-directory .diswork + fileSystem.createDirectories(getJobPath(newJobIntendifier) + "/" + ".diswork"); + log.info("created new job " + newJobIntendifier); + return new JobDescription(newJobIntendifier); + } + + public void submitJob(JobDescription jobDescription) throws DisworkException { + submitJob(jobDescription, new HashMap<String, InputStream>()); + } + + public void submitJob(JobDescription jobDescription, + Map<String, InputStream> inputFiles) throws DisworkException { + + // check dependencies, throw exception + + if (inputFiles.size() + jobDescription.getStagingInputUrls().size() + < jobDescription.getStagingInput().size()) { + // dependencies are missing + } + + try { + String dependencyPath = + getPathForDependency(jobDescription.getApplicationName(), + jobDescription.getApplicationVersion()); + log.info("looking for " + dependencyPath); + + if (!fileSystem.exists(dependencyPath)) { + throw new DisworkException("job require a dependency " + + jobDescription.getApplicationName() + "-" + + jobDescription.getApplicationVersion() + " that is " + + "not available"); + } + + String jobDir = getJobPath(jobDescription); + + if(!fileSystem.exists(jobDir)) { + // strange ! + } + + fileSystem.write(jobDir + "/" + LOG_PATH, IOUtils.toInputStream("")); + + InputStream jobJSDL = IOUtils.toInputStream(jobDescription.toJSDL()); + fileSystem.write(jobDir + "/" + JSDL_PATH, jobJSDL); + + // file staging + for (String fileName : inputFiles.keySet()) { + fileSystem.write(jobDir + "/" + fileName, inputFiles.get(fileName)); + } + + // propose job + String linkName = newJobLinkName(); + + // FIXME 20100609 bleny may throws exception if jobs are proposed + // at a same time + fileSystem.createSymbolicLink(TODO + "/" + linkName, jobDir); + + } catch (IOException e) { + log.error("file system error", e); + throw new DisworkFileSystemException("file system error", e); + } + } + + public boolean checkLogContains(JobDescription job, + String pattern) throws DisworkException { + try { + String jobPath = getJobPath(job); + List<?> entries = IOUtils.readLines(fileSystem.read(jobPath + "/" + LOG_PATH)); + return entries.contains(pattern); + } catch (FileNotFoundException e) { + log.info("log file was not found in job " + job, e); + throw new DisworkException("log file was not found in job " + job, e); + } catch (IOException e) { + log.info("file system error ", e); + throw new DisworkException("file system error ", e); + } + } + + public boolean isFinished(JobDescription job) throws DisworkException { + return checkLogContains(job, "FINISHED"); + } + + public boolean isSuccessful(JobDescription job) throws DisworkException { + return checkLogContains(job, "DONE"); + } + + public boolean isFailed(JobDescription job) throws DisworkException { + return isFinished(job) && !isSuccessful(job); + } + + @Override + public void close() throws IOException { + workers.stop(); + fileSystem.close(); + } + + public Map<String, InputStream> getResults(JobDescription job) + throws DisworkException { + Map<String, InputStream> results = new HashMap<String, InputStream>(); + for (String fileName : job.getStagingOutput()) { + String jobPath = getJobPath(job); + try { + InputStream result = fileSystem.read(jobPath + "/" + fileName); + results.put(fileName, result); + } catch (FileNotFoundException e) { + throw new DisworkException("an expected file is missing", e); + } catch (IOException e) { + log.info("file system error ", e); + throw new DisworkException("file system error ", e); + } + } + return results; + } +} \ No newline at end of file Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,48 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * + * @author bleny + */ +public class DisworkDaemonRunner { + + private static final Log log = LogFactory.getLog(DisworkDaemonRunner.class); + + /** + * @param args + */ + public static void main(String[] args) { + + // consider args + + // DisworkDaemon node = new DisworkDaemon(config); + + } +} Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkException.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,47 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +/** + * + * @author bleny + */ +public class DisworkException extends Exception { + + private static final long serialVersionUID = -6434751198109021511L; + + public DisworkException(String message, Throwable cause) { + super(message, cause); + } + + public DisworkException(String message) { + super(message); + } + + public DisworkException(Throwable cause) { + super(cause); + } + +} Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,47 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +/** + * + * @author bleny + */ +public class DisworkFileSystemException extends DisworkException { + + private static final long serialVersionUID = -4027003687525235092L; + + public DisworkFileSystemException(String message, Throwable cause) { + super(message, cause); + } + + public DisworkFileSystemException(String message) { + super(message); + } + + public DisworkFileSystemException(Throwable cause) { + super(cause); + } + +} Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,185 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +import java.io.Serializable; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * + * + * @author bleny + */ +public class JobDescription implements Serializable { + + private static final long serialVersionUID = -8493700934802808925L; + + /** an id for diswork */ + protected String jobId; + + protected String jobName; + + protected String applicationName; + protected String applicationVersion; + + protected String commandLine; + + /** all files expected at the beginning of the job */ + protected List<String> stagingInput = new ArrayList<String>(); + + /** all files expected at the end of the job */ + protected List<String> stagingOutput = new ArrayList<String>(); + + /** the name of a file and the URI where to get it */ + protected Map<String, URL> stagingInputUrls = new HashMap<String, URL>(); + + /** file where to read the standard input, may be null */ + protected String standardInput; + + /** file where to write the standard output */ + protected String standardOutput; + + /** TOKENS are piece of string you can use for writing command lines */ + protected static Map<String, String> TOKENS; + + static { + TOKENS = new HashMap<String, String>(); + + TOKENS.put("%java", System.getProperty("java.home") + "/bin/java"); + } + + /** + * constructor is protected to prevent bad jobIds. To get a JobDescription + * instance, a client should use the {@link DisworkDaemon#newJob()} + * factory method. The given instance will have a valid jobId + * @param jobId + */ + protected JobDescription(String jobId) { + this.jobId = jobId; + } + + public String getJobId() { + return jobId; + } + + public String getCommandLine() { + String result = commandLine; + for (String token : TOKENS.keySet()) { + result = result.replace(token, TOKENS.get(token)); + } + return result; + } + + public void setCommandLine(String commandLine) { + this.commandLine = commandLine; + } + + public String getJobName() { + return jobName; + } + + public String getApplicationName() { + return applicationName; + } + + public String getApplicationVersion() { + return applicationVersion; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public void setApplication(String applicationName, + String applicationVersion) { + this.applicationName = applicationName; + this.applicationVersion = applicationVersion; + } + + @Override + public String toString() { + return "job : " + jobName + " (" + jobId + ")"; + } + + protected static Integer count = 0; + protected static Map<String, JobDescription> map = new HashMap<String, JobDescription>(); + + public String toJSDL() { + count += 1; + map.put(count.toString(), this); + return count.toString(); + } + + public static JobDescription parseJSDL(String jsdl) { + return map.get(jsdl); + } + + public void addStagingInput(String fileName, URL source) { + stagingInput.add(fileName); + stagingInputUrls.put(fileName, source); + } + + public void addStagingInput(String fileName) { + stagingInput.add(fileName); + } + + public void addStagingOutput(String fileName) { + stagingOutput.add(fileName); + } + + public List<String> getStagingInput() { + return stagingInput; + } + + public List<String> getStagingOutput() { + return stagingOutput; + } + + public Map<String, URL> getStagingInputUrls() { + return stagingInputUrls; + } + + public String getStandardInput() { + return standardInput; + } + + public void setStandardInput(String fileName) { + standardInput = fileName; + } + + public String getStandardOutput() { + return standardOutput; + } + + public void setStandardOutput(String fileName) { + standardOutput = fileName; + } + +} \ No newline at end of file Added: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java (rev 0) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,278 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.daemon; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.util.ZipUtil; + +/** + * + * @author bleny + */ +public class WorkersManager { + + protected static Map<String, String> RUNNING_MOVE = new HashMap<String, String>(); + protected static Map<String, String> FAILED_MOVE = new HashMap<String, String>(); + + // TODO 20100611 bleny make it configurable + /** time to wait beetween two look for a job */ + protected static final int JOB_WAIT = 10 * 1000; + + static { + RUNNING_MOVE.put(DisworkDaemon.TODO, DisworkDaemon.TODO_RUNNING); + RUNNING_MOVE.put(DisworkDaemon.FAILED_1, DisworkDaemon.FAILED_1_RUNNING); + RUNNING_MOVE.put(DisworkDaemon.FAILED_2, DisworkDaemon.FAILED_2_RUNNING); + + RUNNING_MOVE.put(DisworkDaemon.TODO, DisworkDaemon.FAILED_1); + RUNNING_MOVE.put(DisworkDaemon.FAILED_1, DisworkDaemon.FAILED_2); + RUNNING_MOVE.put(DisworkDaemon.FAILED_2, DisworkDaemon.FAILED_3); + } + + private static final Log log = LogFactory.getLog(WorkersManager.class); + + protected DisworkFileSystem fileSystem; + protected DisworkConfig config; + + // Pool of workers ? + protected Worker worker; + + // Activity strategy ? + + protected class Worker implements Runnable { + + public boolean shouldStop = false; + + protected void log(String jobPath, String message) throws IOException { + String logPath = jobPath + "/" + DisworkDaemon.LOG_PATH; + InputStream oldLogAsStream = fileSystem.read(logPath); + String oldLog = IOUtils.toString(oldLogAsStream); + + String logEntry = message + "\n"; + + String newLog = oldLog + logEntry; + + fileSystem.delete(logPath); + fileSystem.write(logPath, IOUtils.toInputStream(newLog)); + } + + protected boolean runJob(String jobPath) throws IOException { + log.info("running job at " + jobPath); + + String jsdl; + try { + String jsdlPath = jobPath + "/" + DisworkDaemon.JSDL_PATH; + jsdl = IOUtils.toString(fileSystem.read(jsdlPath)); + } catch (FileNotFoundException e) { + log.warn("job " + jobPath + " misses a job description"); + return false; + } + + log.info("read jsdl " + jsdl); + + JobDescription jobDescription = JobDescription.parseJSDL(jsdl); + + log.info("will run job " + jobDescription); + + // create temp dir + File jobDir = new File(config.getTempDirectory(), jobDescription.getJobId()); + jobDir.mkdirs(); + + // download application + String applicationPath = DisworkDaemon.getPathForDependency( + jobDescription.getApplicationName(), + jobDescription.getApplicationVersion()); + InputStream applicationData = fileSystem.read(applicationPath); + + + File application = new File(jobDir, FilenameUtils.getName(applicationPath)); + application.createNewFile(); + log.info("will create " + application.getAbsolutePath()); + OutputStream out = new FileOutputStream(application); + IOUtils.copy(applicationData, out); + // unzip application + ZipUtil.uncompress(application, jobDir); + + // staging input files + for (String fileName : jobDescription.getStagingInput()) { + File localCopy = new File(jobDir, fileName); + localCopy.createNewFile(); + InputStream source = null; + if (jobDescription.getStagingInputUrls().containsKey(fileName)) { + // download this file from URL + URL url = jobDescription.getStagingInputUrls().get(fileName); + log.info("downloading from " + url); + source = url.openStream(); + } else { + // download this file from diswork + source = fileSystem.read(jobPath + "/" + fileName); + } + IOUtils.copy(source, new FileOutputStream(localCopy)); + } + + // executing the job + String commandLine = jobDescription.getCommandLine(); + log.info("calling " + commandLine); + String[] bidule = commandLine.split(" "); + ProcessBuilder builder = new ProcessBuilder(bidule); + builder.directory(jobDir); + builder.redirectErrorStream(true); + Process job = builder.start(); + + // plugin a file on the standard input + String standardInputFileName = jobDescription.getStandardInput(); + if (standardInputFileName != null) { + InputStream input = new FileInputStream(new File(jobDir, standardInputFileName)); + IOUtils.copy(input, job.getOutputStream()); + } + + int exitValue = -1; + try { + exitValue = job.waitFor(); + } catch (InterruptedException e) { + log.error("job " + jobDescription + " was interupted", e); + + // FIXME 20100611 bleny job is considered has failed + exitValue = 1; + } + + // dump the standard output in a file + String standardOutputFileName = jobDescription.getStandardOutput(); + if (standardOutputFileName != null) { + OutputStream output = new FileOutputStream(new File(jobDir, standardOutputFileName)); + IOUtils.copy(job.getInputStream(), output); + } + + log.info("job returned " + exitValue); + + // output file staging + for (String fileName : jobDescription.getStagingOutput()) { + File localCopy = new File(jobDir, fileName); + InputStream localCopyStream = new FileInputStream(localCopy); + + String filePath = jobPath + "/" + fileName; + + log.info("out-staging " + fileName); + // erase before write + if (fileSystem.exists(filePath)) { + fileSystem.delete(filePath); + } + + fileSystem.write(filePath, localCopyStream); + localCopyStream.close(); + } + + // clean up the job directory + // FileUtil.deleteRecursively(jobDir); + + boolean success = exitValue == 0; + if (success) { + log(jobPath, "DONE\nFINISHED"); + } else { + log(jobPath, "FAILED\nFINISHED"); + } + return success; + } + + @Override + public void run() { + while (! shouldStop) { + // try to find a new job + try { + // TODO 20100609 bleny watch for other jobs + List<String> jobsNames = + fileSystem.readDirectory(DisworkDaemon.TODO); + if (jobsNames.size() != 0) { + // sort and choose the first, due to names, it should be + // the more ancient one + Collections.sort(jobsNames); + String oldName = jobsNames.get(0); + + String newName = DisworkDaemon.newJobLinkName(); + fileSystem.move(DisworkDaemon.TODO + "/" + oldName, + DisworkDaemon.TODO_RUNNING + "/" + newName); + + boolean jobSuccess = runJob(DisworkDaemon.TODO_RUNNING + "/" + newName); + + oldName = newName; + newName = DisworkDaemon.newJobLinkName(); + + if (jobSuccess) { + fileSystem.move( + DisworkDaemon.TODO_RUNNING + "/" + oldName, + DisworkDaemon.DONE + "/" + newName); + } else { + fileSystem.move( + DisworkDaemon.TODO_RUNNING + "/" + oldName, + DisworkDaemon.FAILED_3 + "/" + newName); + } + } else { + log.info("nothing to do"); + Thread.sleep(JOB_WAIT); + } + + } catch (IOException e) { + log.error("error while reading jobs", e); + // TODO 20100611 bleny manage exception + } catch (InterruptedException e) { + log.info("exception catch", e); + // TODO 20100611 bleny manage exception + } + } + } + + } + + public WorkersManager(DisworkFileSystem fileSystem, DisworkConfig config) { + this.fileSystem = fileSystem; + this.config = config; + } + + public void start() { + worker = new Worker(); + Thread t = new Thread(worker); + t.start(); + } + + public void stop() { + worker.shouldStop = true; + } +} \ No newline at end of file Added: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java (rev 0) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,106 @@ +package org.nuiton.diswork.daemon; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.InputStream; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class DisworkDaemonTest { + + protected DisworkDaemon daemon; + + protected static int port = 39999; + + @Before + public void setUp() throws Exception { + DisworkConfig config = DisworkConfig.newConfig(); + port += 1; + config.setUsedPort(port); + daemon = new DisworkDaemon(config); + InputStream application = ClassLoader.getSystemResourceAsStream("fake-app-1.0.zip"); + daemon.submitApplication("fake-app", "1.0", application); + } + + @After + public void tearDown() throws Exception { + daemon.close(); + } + + @Test(expected = DisworkException.class) + public void testSubmitWithoutDependency() throws Exception { + JobDescription job = daemon.newJob(); + job.setApplication("non-existing-application", "0.0"); + daemon.submitJob(job); + } + + @Test + public void testSubmitSuccessfulJob() throws Exception { + JobDescription job = daemon.newJob(); + job.setApplication("fake-app", "1.0"); + job.setCommandLine("%java -jar fake-app.jar"); + daemon.submitJob(job); + + while(! daemon.isFinished(job)) { + Thread.sleep(5 * 1000); + } + + assertTrue(daemon.isSuccessful(job)); + } + + @Test + public void testSubmitFailJob() throws Exception { + JobDescription job = daemon.newJob(); + job.setApplication("fake-app", "1.0"); + job.setCommandLine("%java -jar fake-app.jar fail"); + daemon.submitJob(job); + + while(! daemon.isFinished(job)) { + Thread.sleep(5 * 1000); + } + + assertTrue(daemon.isFailed(job)); + } + + @Test + public void testStaging() throws Exception { + JobDescription job = daemon.newJob(); + job.setJobName("My Job"); + job.setApplication("fake-app", "1.0"); + job.setCommandLine("%java -jar fake-app.jar"); + + job.addStagingInput("example.com_index", new URL("http://www.example.com/")); + job.addStagingInput("input.txt"); + job.addStagingOutput("output.txt"); + + job.setStandardInput("input.txt"); + job.setStandardOutput("output.txt"); + + Map<String, InputStream> data = new HashMap<String, InputStream>(); + data.put("input.txt", ClassLoader.getSystemResourceAsStream("input.txt")); + + daemon.submitJob(job, data); + + while(! daemon.isFinished(job)) { + Thread.sleep(1 * 1000); + } + + assertTrue(daemon.isSuccessful(job)); + + Map<String, InputStream> results = daemon.getResults(job); + + assertTrue(results.containsKey("output.txt")); + + String output = IOUtils.toString(results.get("output.txt")); + assertEquals("a print on standard output\n", output); + + } + +} Added: trunk/diswork-daemon/src/test/resources/fake-app-1.0.zip =================================================================== (Binary files differ) Property changes on: trunk/diswork-daemon/src/test/resources/fake-app-1.0.zip ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: trunk/diswork-daemon/src/test/resources/fake-app.jar =================================================================== (Binary files differ) Property changes on: trunk/diswork-daemon/src/test/resources/fake-app.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: trunk/diswork-daemon/src/test/resources/input.txt =================================================================== --- trunk/diswork-daemon/src/test/resources/input.txt (rev 0) +++ trunk/diswork-daemon/src/test/resources/input.txt 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1 @@ +Hello Added: trunk/diswork-daemon/src/test/resources/log4j.properties =================================================================== --- trunk/diswork-daemon/src/test/resources/log4j.properties (rev 0) +++ trunk/diswork-daemon/src/test/resources/log4j.properties 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,9 @@ +# Global logging configuration +log4j.rootLogger=WARN, stdout +# Console output... +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n +# package level +log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO +log4j.logger.org.nuiton.diswork.daemon=INFO \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-11 15:52:09 UTC (rev 73) @@ -111,6 +111,7 @@ List<String> todos = fileSystem.readDirectory("/todo"); if (todos.isEmpty()) { log.info("nothing to do"); + System.out.println("nothing to do"); } else { // taking a random job Random random = new Random(); Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-11 15:52:09 UTC (rev 73) @@ -680,7 +680,7 @@ */ protected void checkPathSyntax(String path) throws IOException { if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { - throw new IOException("\" + path + \" is not correct, all pathes " + + throw new IOException("\"" + path + "\" is not correct, all pathes " + "have to be absolute (thus, starts with)" + EntryUtil.ROOT_DIRECTORY); } @@ -688,7 +688,7 @@ String doubleSeparator = EntryUtil.PATH_SEPARATOR + EntryUtil.PATH_SEPARATOR; if (path.contains(doubleSeparator)) { - throw new IOException("\" + path + \" is not correct, it contains " + throw new IOException("\"" + path + "\" is not correct, it contains " + doubleSeparator); } } @@ -809,4 +809,21 @@ } } + + public void createDirectories(String path) + throws ConcurrentModificationException, + IOException { + log.info("trying create directories for " + path); + + String pathWithoutRoot = path.substring(1, path.length()); + + String[] dirs = pathWithoutRoot.split(EntryUtil.PATH_SEPARATOR); + String dirPath = ""; + for (String dir : dirs) { + dirPath += EntryUtil.PATH_SEPARATOR + dir; + if (!exists(dirPath)) { + createDirectory(dirPath); + } + } + } } \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-11 15:52:09 UTC (rev 73) @@ -81,8 +81,15 @@ * @return * @throws UnknownHostException */ - public static String getIp() throws UnknownHostException { - InetAddress result = InetAddress.getLocalHost(); + public static String getIp() { + InetAddress result = null; + try { + result = InetAddress.getLocalHost(); + } catch (UnknownHostException e) { + // LocalHost is always known + log.error(e); + } + if (result.isLoopbackAddress()) { try { Socket temp = new Socket("microsoft.com", 80); @@ -140,8 +147,7 @@ * @return * @throws UnknownHostException */ - public static DisworkFileSystemConfig newPastryDisworkConfig() - throws UnknownHostException { + public static DisworkFileSystemConfig newPastryDisworkConfig() { return newPastryDisworkConfig(null); } @@ -153,8 +159,8 @@ * @return a complete config * @throws UnknownHostException */ - public static DisworkFileSystemConfig newPastryDisworkConfig(Integer bootstrapPort) - throws UnknownHostException { + public static DisworkFileSystemConfig + newPastryDisworkConfig(Integer bootstrapPort) { DisworkFileSystemConfig result = new DisworkFileSystemConfig(); String port = getPort().toString(); String ip = getIp(); @@ -169,13 +175,12 @@ return result; } - public static DisworkFileSystemConfig newKademliaDisworkConfig() - throws UnknownHostException { + public static DisworkFileSystemConfig newKademliaDisworkConfig() { return newKademliaDisworkConfig(null); } - public static DisworkFileSystemConfig newKademliaDisworkConfig(Integer bootstrapPort) - throws UnknownHostException { + public static DisworkFileSystemConfig + newKademliaDisworkConfig (Integer bootstrapPort) { DisworkFileSystemConfig result = new DisworkFileSystemConfig(); String port = getPort().toString(); String ip = getIp(); Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java 2010-06-11 15:52:09 UTC (rev 73) @@ -39,6 +39,8 @@ public class InMemoryDisworkMap extends HashMap<String, byte[]> implements DisworkMap { + private static final long serialVersionUID = 2467699175129909832L; + private final Log log = LogFactory.getLog(InMemoryDisworkMap.class); @Override Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-06-11 15:52:09 UTC (rev 73) @@ -49,14 +49,15 @@ protected Kademlia kad; public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { - log.info("booting kademlia dht on port " + config.getUsedPort()); + kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); - + if (config.getBootstrapIp() != null) { InetSocketAddress bootstrap = new InetSocketAddress( config.getBootstrapIp(), config.getBootstrapPort()); try { + log.info("trying to connect to " + bootstrap); kad.connect(bootstrap); } catch (RoutingException e) { log.error("bootstrap node is unreachable", e); @@ -64,6 +65,8 @@ } } + log.info("kademlia status : " + kad); + //Identifier.IDSIZE = 1024; log.info("using " + Identifier.IDSIZE + " bytes for identifiers"); } @@ -93,7 +96,7 @@ } - log.info("key for string " + s + " is " + id); + log.debug("key for string " + s + " is " + id); return id; } Deleted: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-11 15:52:09 UTC (rev 73) @@ -1,9 +0,0 @@ -# Global logging configuration -log4j.rootLogger=WARN, stdout -# Console output... -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n -# package level -log4j.logger.org.nuiton.diswork.fs=WARN -log4j.logger.org.nuiton.diswork.fs.Demo=INFO \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-11 15:52:09 UTC (rev 73) @@ -394,4 +394,10 @@ } */ } + + @Test + public void testCreateDirectories() throws Exception { + fileSystem.createDirectories("/dir/subdir/subsubdir"); + assertTrue(fileSystem.exists("/dir/subdir/subsubdir")); + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-11 15:52:09 UTC (rev 73) @@ -27,7 +27,7 @@ // finally, initiate the fileSystem DisworkFileSystemConfig disworkConfig1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); - bootstrapPort = disworkConfig1.getUsedPort(); + bootstrapPort = disworkConfig1.getUsedPort() + 1; fileSystem = new DisworkFileSystem(disworkConfig1); } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-10 09:32:59 UTC (rev 72) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-11 15:52:09 UTC (rev 73) @@ -3,7 +3,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -12,7 +11,6 @@ import org.junit.Before; import org.junit.Test; import org.nuiton.diswork.fs.DisworkFileSystemConfig; -import org.nuiton.diswork.fs.storage.KademliaDisworkMap; import org.planx.xmlstore.routing.Identifier; public class KademliaDisworkMapTest extends AbstractDisworkMapTest { @@ -24,7 +22,7 @@ map1 = new KademliaDisworkMap(config1); DisworkFileSystemConfig config2 = DisworkFileSystemConfig - .newKademliaDisworkConfig(config1.getUsedPort()); + .newKademliaDisworkConfig(config1.getUsedPort()+1); map2 = new KademliaDisworkMap(config2); } @@ -72,7 +70,7 @@ * a bad bootstrap * @throws Exception */ - @Test(expected = IOException.class) + @Test(expected = org.planx.xmlstore.routing.RoutingException.class) public void testBadBootrap() throws Exception { DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); Copied: trunk/diswork-fs/src/test/resources/log4j.properties (from rev 72, trunk/diswork-fs/src/main/resources/log4j.properties) =================================================================== --- trunk/diswork-fs/src/test/resources/log4j.properties (rev 0) +++ trunk/diswork-fs/src/test/resources/log4j.properties 2010-06-11 15:52:09 UTC (rev 73) @@ -0,0 +1,10 @@ +# Global logging configuration +log4j.rootLogger=WARN, stdout +# Console output... +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n +# package level +log4j.logger.org.nuiton.diswork.fs=WARN +log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO +log4j.logger.org.nuiton.diswork.fs.Demo=INFO \ No newline at end of file