June 2010 - Diswork-commits - mailman.cloud.codelutin.com

r72 - trunk/diswork-fs/src/main/resources
by bleny＠users.nuiton.org 10 Jun '10

10 Jun '10

Author: bleny Date: 2010-06-10 11:32:59 +0200 (Thu, 10 Jun 2010) New Revision: 72 Url: http://nuiton.org/repositories/revision/diswork/72 Log: fs : update log4j.prop Modified: trunk/diswork-fs/src/main/resources/log4j.properties Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-10 09:30:17 UTC (rev 71) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-10 09:32:59 UTC (rev 72) @@ -5,6 +5,5 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=WARN -#log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO -log4j.logger.org.nuiton.disworkfs.Demo=INFO \ No newline at end of file +log4j.logger.org.nuiton.diswork.fs=WARN +log4j.logger.org.nuiton.diswork.fs.Demo=INFO \ No newline at end of file

1 0

r71 - in trunk/diswork-fs/src: main/java/org/nuiton/diswork/fs main/java/org/nuiton/diswork/fs/storage main/resources test/java/org/nuiton/diswork/fs test/java/org/nuiton/diswork/fs/storage
by bleny＠users.nuiton.org 10 Jun '10

10 Jun '10

Author: bleny Date: 2010-06-10 11:30:17 +0200 (Thu, 10 Jun 2010) New Revision: 71 Url: http://nuiton.org/repositories/revision/diswork/71 Log: implementation move + test, gestion de l'echec du bootstrap dans kademlia + test, traces dans la demo Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-09 08:40:08 UTC (rev 70) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-10 09:30:17 UTC (rev 71) @@ -90,6 +90,7 @@ } } catch (IOException e) { log.error(e); + System.out.println("error occured " + e); } try { Thread.sleep(15 * 1000); @@ -142,11 +143,13 @@ } } catch (IOException e) { log.error(e); + System.out.println("error occured " + e); } try { Thread.sleep(10 * 1000); } catch (InterruptedException e) { log.info("sleep interrupted", e); + System.out.println("error occured " + e); } } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-09 08:40:08 UTC (rev 70) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-10 09:30:17 UTC (rev 71) @@ -692,4 +692,121 @@ + doubleSeparator); } } + + /** + * move a file, a directory a link. May be used for renaming purpose + * @param path the path to the object to move + * @param destination the full path of the target + * @throws IOException + */ + public void move(String path, String destination) throws IOException { + checkPathSyntax(path); + checkPathSyntax(destination); + + String pathParent = EntryUtil.getParentFromPath(path); + String pathName = EntryUtil.getNameFromPath(path); + String destinationParent = EntryUtil.getParentFromPath(destination); + String destinationName = EntryUtil.getNameFromPath(destination); + + move(pathParent, pathName, destinationParent, destinationName); + + } + + protected void move(String pathParent, String pathName, + String destinationParent, String destinationName) throws IOException { + + String entryParent = walk(pathParent); + + if (entryParent == null) { + throw new IOException(pathParent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + + String destinationParentEntry = walk(destinationParent); + if (destinationParentEntry == null) { + throw new IOException(destinationParentEntry + " directory" + + "doesn't exists"); + } + + if (EntryUtil.isDirectory(destinationParentEntry)) { + + // we are now in actual directories for old path and destination + + String parentContent = storage.getDirectory( + EntryUtil.getIdFromEntry(entryParent)); + String oldEntry = EntryUtil.findEntryInDirectory(parentContent, + pathName); + if (oldEntry == null) { + throw new IOException("no element " + pathName + " in " + + pathParent); + } + + String parentId = EntryUtil.getIdFromEntry(entryParent); + String destinationParentId = EntryUtil.getIdFromEntry(destinationParentEntry); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId) + && storage.tryToLock(destinationParentId); + if (lockAcquired) { + // we have locked, do the update + parentContent = storage.getDirectory(parentId); + String newContent = EntryUtil.removeEntryFromEntries(parentContent, pathName); + storage.putDirectory(parentId, newContent); + + + String destinationParentContent = storage.getDirectory(destinationParentId); + newContent = EntryUtil.addEntryToDirectoryContent(destinationParentContent, + EntryUtil.getTypeFromEntry(oldEntry), + destinationName, + EntryUtil.getIdFromEntry(oldEntry)); + storage.putDirectory(destinationParentId, newContent); + + storage.unLock(parentId); + storage.unLock(destinationParentId); + } else { + log.info(pathParent + " is locked"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + pathParent + " directory is locked"); + } + + } else if (EntryUtil.isLink(destinationParentEntry)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(destinationParentEntry)); + String newTarget = EntryUtil.resolveLink(destinationParent, linkTarget); + move(newTarget, pathName, destinationParent, destinationName); + } else if (EntryUtil.isFile(destinationParentEntry)) { + throw new IOException(destinationParent + " is not a directory"); + } else { + log.warn("strange entry" + destinationParentEntry); + throw new IOException("strange entry" + destinationParentEntry); + } + + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(pathParent, linkTarget); + move(pathParent, pathName, newTarget, destinationName); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(pathParent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + + } } \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-06-09 08:40:08 UTC (rev 70) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-06-10 09:30:17 UTC (rev 71) @@ -40,6 +40,7 @@ import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.planx.xmlstore.routing.Identifier; import org.planx.xmlstore.routing.Kademlia; +import org.planx.xmlstore.routing.RoutingException; public class KademliaDisworkMap implements DisworkMap { @@ -48,13 +49,19 @@ protected Kademlia kad; public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { + log.info("booting kademlia dht on port " + config.getUsedPort()); kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); if (config.getBootstrapIp() != null) { InetSocketAddress bootstrap = new InetSocketAddress( config.getBootstrapIp(), config.getBootstrapPort()); - kad.connect(bootstrap); + try { + kad.connect(bootstrap); + } catch (RoutingException e) { + log.error("bootstrap node is unreachable", e); + throw e; + } } //Identifier.IDSIZE = 1024; Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-09 08:40:08 UTC (rev 70) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-10 09:30:17 UTC (rev 71) @@ -6,4 +6,5 @@ log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level log4j.logger.org.nuiton.disworkfs=WARN +#log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO log4j.logger.org.nuiton.disworkfs.Demo=INFO \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-09 08:40:08 UTC (rev 70) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-10 09:30:17 UTC (rev 71) @@ -327,7 +327,7 @@ } @Test - public void testLinks() throws Exception { + public void testLinks() throws Exception { fileSystem.createDirectory("/dir"); fileSystem.createDirectory("/dir/subdir"); fileSystem.createDirectory("/otherdir"); @@ -358,4 +358,40 @@ assertEquals(1, readResult.size()); assertTrue(readResult.contains("subdir")); } + + @Test + public void testMove() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/dir/subdir/subsubdir"); + fileSystem.createSymbolicLink("/dir/link", "subdir"); + fileSystem.write("/dir/file", new FileInputStream(randomFilePath)); + + fileSystem.createDirectory("/otherdir"); + fileSystem.move("/dir/subdir", "/otherdir/subdir"); + fileSystem.move("/dir/link", "/otherdir/link"); + fileSystem.move("/dir/file", "/otherdir/newfile"); + + List<String> readDir = fileSystem.readDirectory("/dir"); + for (String c : readDir) { + System.out.println(c); + } + assertTrue(readDir.isEmpty()); + + readDir = fileSystem.readDirectory("/otherdir"); + assertEquals(3, readDir.size()); + + assertTrue(fileSystem.exists("/otherdir/newfile")); + assertTrue(fileSystem.exists("/otherdir/subdir/subsubdir")); + + // FIXME 20100610 bleny after moving a link, it still points the old target + + //assertTrue(fileSystem.exists("/otherdir/link/subsubdir")); + /* + readDir = fileSystem.readDirectory("/otherdir/link"); + for (String e : readDir) { + System.out.println(e); + } + */ + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-09 08:40:08 UTC (rev 70) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-10 09:30:17 UTC (rev 71) @@ -3,6 +3,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -18,10 +19,12 @@ @Before public void setUp() throws Exception { - DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); + DisworkFileSystemConfig config1 = + DisworkFileSystemConfig.newKademliaDisworkConfig(); map1 = new KademliaDisworkMap(config1); - DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); + DisworkFileSystemConfig config2 = DisworkFileSystemConfig + .newKademliaDisworkConfig(config1.getUsedPort()); map2 = new KademliaDisworkMap(config2); } @@ -63,4 +66,18 @@ // if keys set is too small, there are collisions assertEquals(paths.size(), keys.size()); } + + /** + * Tests that in exception is raised when attempting to connect using + * a bad bootstrap + * @throws Exception + */ + @Test(expected = IOException.class) + public void testBadBootrap() throws Exception { + DisworkFileSystemConfig config1 = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config1.setBootstrapIp("microsoft.com"); + config1.setBootstrapPort(80); + new KademliaDisworkMap(config1); + } }

1 0

r70 - in trunk/diswork-fs/src: main/java/org/nuiton/diswork/fs main/java/org/nuiton/diswork/fs/storage test/java/org/nuiton/diswork/fs
by bleny＠users.nuiton.org 09 Jun '10

09 Jun '10

Author: bleny Date: 2010-06-09 10:40:08 +0200 (Wed, 09 Jun 2010) New Revision: 70 Url: http://nuiton.org/repositories/revision/diswork/70 Log: demo avec traces ; style Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-08 14:35:06 UTC (rev 69) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-09 08:40:08 UTC (rev 70) @@ -63,6 +63,7 @@ InputStream source = IOUtils.toInputStream(job); log.info("writing job " + waitingForJob + " (" + randomInteger + ")"); + System.out.println("submited job " + randomInteger); fileSystem.write(waitingForJob, source); source.close(); } else { @@ -76,6 +77,8 @@ Integer result = Integer.parseInt(resultString); log.info("result is " + result + " (expected : " + expectedResult + ")"); + System.out.println("job result published " + + result); fileSystem.delete(resultPath); fileSystem.delete(waitingForJob); waitingForJob = null; @@ -118,11 +121,15 @@ log.info("reading the job " + jobPath); InputStream in = fileSystem.read(jobPath); String operation = IOUtils.toString(in); - // in.close(); + in.close(); log.info("operation to do " + operation); + System.out.println("job found " + + operation); Integer i = Integer.parseInt(operation); i += 1; String result = i.toString(); + System.out.println("published result " + + result); log.info("result is " + result); InputStream source = IOUtils.toInputStream(result); Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-08 14:35:06 UTC (rev 69) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-09 08:40:08 UTC (rev 70) @@ -449,7 +449,7 @@ } } - private boolean isLockStillOwned(String key) { + protected boolean isLockStillOwned(String key) { String lockKey = keyToLockKey(key); byte[] lock = map.get(lockKey); String lockOwner = EntryUtil.getOwnerFromLock(lock); Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-08 14:35:06 UTC (rev 69) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-09 08:40:08 UTC (rev 70) @@ -25,7 +25,7 @@ import org.nuiton.util.FileUtil; -public abstract class AbstractDisworkFileSystemTest { +public abstract class AbstractDisworkFileSystemTest { /** * a place to store files for the test it's a subdirectory of the OS temp

1 0

r69 - in trunk/diswork-fs: . src/main/java/org/nuiton/diswork/fs src/main/java/org/nuiton/diswork/fs/storage
by bleny＠users.nuiton.org 08 Jun '10

08 Jun '10

Author: bleny Date: 2010-06-08 16:35:06 +0200 (Tue, 08 Jun 2010) New Revision: 69 Url: http://nuiton.org/repositories/revision/diswork/69 Log: fs : syntax check on paths Modified: trunk/diswork-fs/run_demo.sh trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java Modified: trunk/diswork-fs/run_demo.sh =================================================================== --- trunk/diswork-fs/run_demo.sh 2010-06-08 13:16:35 UTC (rev 68) +++ trunk/diswork-fs/run_demo.sh 2010-06-08 14:35:06 UTC (rev 69) @@ -1,2 +1,2 @@ -mvn -e exec:java -Dexec.mainClass=org.nuiton.disworkfs.Demo -Dexec.args="consumer 9001" -Dexec.classpathScope=test -mvn -e exec:java -Dexec.mainClass=org.nuiton.disworkfs.Demo -Dexec.args="producer 9002 127.0.0.1 9001" -Dexec.classpathScope=test +mvn -e exec:java -Dexec.mainClass=org.nuiton.diswork.fs.Demo -Dexec.args="consumer 9001" -Dexec.classpathScope=test +mvn -e exec:java -Dexec.mainClass=org.nuiton.diswork.fs.Demo -Dexec.args="producer 9002 127.0.0.1 9001" -Dexec.classpathScope=test Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-08 13:16:35 UTC (rev 68) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-08 14:35:06 UTC (rev 69) @@ -89,6 +89,7 @@ * @throws IOException */ public boolean exists(String path) throws IOException { + checkPathSyntax(path); String entry = walk(path); boolean result = entry != null; return result; @@ -104,6 +105,7 @@ */ public InputStream read(String path) throws FileNotFoundException, IOException { + checkPathSyntax(path); String entry = walk(path); if (entry == null) { @@ -141,8 +143,9 @@ public void write(String path, InputStream source) throws IOException, ConcurrentModificationException { + checkPathSyntax(path); if (source == null) { - throw new IOException("source is not readable (null stream)"); + throw new NullPointerException("source stream is null"); } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); @@ -224,6 +227,7 @@ public void createDirectory(String path) throws IOException, ConcurrentModificationException { + checkPathSyntax(path); String parent = EntryUtil.getParentFromPath(path); String dirName = EntryUtil.getNameFromPath(path); createDirectory(parent, dirName); @@ -309,6 +313,7 @@ public void createSymbolicLink(String path, String target) throws IOException, ConcurrentModificationException { + checkPathSyntax(path); String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); createSymbolicLink(parent, name, target); @@ -411,6 +416,7 @@ */ public void delete(String path) throws IOException, ConcurrentModificationException { + checkPathSyntax(path); String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); log.info("trying to remove " + path); @@ -510,6 +516,7 @@ * @throws IOException if path doesn't point to a directory */ public List<String> readDirectory(String path) throws IOException { + checkPathSyntax(path); String entry = walk(path); List<String> result = null; @@ -660,11 +667,29 @@ @Override public void close() throws IOException { - storage.close(); + storage.close(); } protected void setMap(DisworkMap map) { storage.setMap(map); } -} + /** + * check a path is absolute and syntactically correct, throw exception if + * that's not the case. + */ + protected void checkPathSyntax(String path) throws IOException { + if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { + throw new IOException("\" + path + \" is not correct, all pathes " + + "have to be absolute (thus, starts with)" + + EntryUtil.ROOT_DIRECTORY); + } + + String doubleSeparator = EntryUtil.PATH_SEPARATOR + + EntryUtil.PATH_SEPARATOR; + if (path.contains(doubleSeparator)) { + throw new IOException("\" + path + \" is not correct, it contains " + + doubleSeparator); + } + } +} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-08 13:16:35 UTC (rev 68) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-08 14:35:06 UTC (rev 69) @@ -51,12 +51,16 @@ protected static final long LOCK_VALID_TIME = 60 * 60 * 1000; + /** this id will be used to sign locks */ protected String ownerId = EntryUtil.generateId(); + /** a key where the lock for key should be put at the returned value */ protected static String keyToLockKey(String key) { return key + "_lock"; } - + + /** when using copy-on-write new data for the value of key should be stored + * to the returned value */ protected static String keyToNewDataKey(String key) { return key + "_new_data"; }

1 0

r68 - in trunk/diswork-fs/src: main/java/org/nuiton main/java/org/nuiton/diswork main/java/org/nuiton/diswork/fs main/java/org/nuiton/diswork/fs/peerunit main/java/org/nuiton/diswork/fs/storage test/java/org/nuiton test/java/org/nuiton/diswork test/java/org/nuiton/diswork/fs test/java/org/nuiton/diswork/fs/storage
by bleny＠users.nuiton.org 08 Jun '10

08 Jun '10

Author: bleny Date: 2010-06-08 15:16:35 +0200 (Tue, 08 Jun 2010) New Revision: 68 Url: http://nuiton.org/repositories/revision/diswork/68 Log: changement du package de org.nuiton.disworkfs en org.nuiton.diswork.fs Added: trunk/diswork-fs/src/main/java/org/nuiton/diswork/ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/ trunk/diswork-fs/src/test/java/org/nuiton/diswork/ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/ Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/ Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/package-info.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/EntryUtilTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/PastryDisworkMapTest.java Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,214 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class Demo { + + private static final Log log = LogFactory.getLog(Demo.class); + + protected static final String USAGE = + "Usage :\n" + + "org.nuiton.disworkfs.Demo producer|consumer" + + " usePort [bootStrapIp bootStrapPort]"; + + protected static DisworkFileSystem fileSystem; + + protected static class Producer implements Runnable { + + protected String waitingForJob = null; + protected Integer expectedResult = null; + + @Override + public void run() { + while (true) { + try { + if (fileSystem.exists("/todo")) { + if (waitingForJob == null) { + Random random = new Random(); + waitingForJob = "/todo/job-" + random.nextInt(); + Integer randomInteger = random.nextInt(); + expectedResult = randomInteger + 1; + String job = randomInteger.toString(); + InputStream source = IOUtils.toInputStream(job); + log.info("writing job " + waitingForJob + " (" + + randomInteger + ")"); + fileSystem.write(waitingForJob, source); + source.close(); + } else { + String resultPath = waitingForJob + ".result"; + if (fileSystem.exists(resultPath)) { + log.info("results has been published"); + InputStream resultStream = + fileSystem.read(resultPath); + String resultString = + IOUtils.toString(resultStream); + Integer result = Integer.parseInt(resultString); + log.info("result is " + result + " (expected : " + + expectedResult + ")"); + fileSystem.delete(resultPath); + fileSystem.delete(waitingForJob); + waitingForJob = null; + expectedResult = null; + } + } + } else { + fileSystem.createDirectory("/todo"); + } + } catch (IOException e) { + log.error(e); + } + try { + Thread.sleep(15 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + } + } + } + + protected static class Consumer implements Runnable { + + @Override + public void run() { + while (true) { + try { + if (fileSystem.exists("/todo")) { + List<String> todos = fileSystem.readDirectory("/todo"); + if (todos.isEmpty()) { + log.info("nothing to do"); + } else { + // taking a random job + Random random = new Random(); + String todo = todos.get( + random.nextInt(todos.size()) + ); + if (!todo.endsWith(".result")) { + String jobPath = "/todo/" + todo; + log.info("reading the job " + jobPath); + InputStream in = fileSystem.read(jobPath); + String operation = IOUtils.toString(in); + // in.close(); + log.info("operation to do " + operation); + Integer i = Integer.parseInt(operation); + i += 1; + String result = i.toString(); + log.info("result is " + result); + InputStream source = + IOUtils.toInputStream(result); + fileSystem.write(jobPath + ".result", source); + source.close(); + } + } + } else { + fileSystem.createDirectory("/todo"); + } + } catch (IOException e) { + log.error(e); + } + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + } + } + } + + public static void printUsage() { + System.out.println(USAGE); + } + + /** + * + * <dl> + * <dt>producer|consumer (required)</dt> + * <dd>a producer will produce operations todo, a consumer may read + * operations and try to execute them + * </dd> + * <dt>port (required)</dt> + * <dd>the port to use</dd> + * <dt>bootstrapip (optional)</dt> + * <dd>if not provided, the node will bootstrap itself, creating a + * new network. If provided, the node will first try to join a + * network</dd> + * <dt>bootstrap port (required if bootstrapip is set)</dt> + * <dd>the port used by the node to join</dd> + * </dl> + * + * Example : + * org.nuiton.disworkfs.Demo consumer 9001 + * org.nuiton.disworkfs.Demo producer 9002 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9003 127.0.0.1 9002 + * org.nuiton.disworkfs.Demo producer 9004 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9005 127.0.0.1 9003 + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + if (args.length == 2) { + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setOption("diswork.fs.use_port", args[1]); + fileSystem = new DisworkFileSystem(config); + if ("producer".equals(args[0])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[0])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + } else if (args.length == 4) { + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setOption("diswork.fs.use_port", args[1]); + config.setOption("diswork.fs.bootstrap.ip", args[2]); + config.setOption("diswork.fs.bootstrap.port", args[3]); + fileSystem = new DisworkFileSystem(config); + if ("producer".equals(args[0])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[0])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + } else { + printUsage(); + } + } +} \ No newline at end of file Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,670 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.ConcurrentModificationException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.diswork.fs.storage.DisworkMap; +import org.nuiton.diswork.fs.storage.EntryUtil; +import org.nuiton.diswork.fs.storage.Storage; + +/** Main class of Diswork File System, provide methods for all operations + * You can use: + * <ul> + * <li>{@link #createDirectory(String)} and {@link #readDirectory(String)} + * to create and browse directories</li> + * <li>{@link #write(String, InputStream)} and {@link #read(String)} + * to write a file and read it</li> + * <li>{@link #createSymbolicLink(String, String)} to create symbolic links</li> + * <li>{@link #exists(String)} and {@link #delete(String)} can be used + * on directories, files and symlinks</li> + * </ul> + */ +public class DisworkFileSystem implements Closeable { + + private static final Log log = LogFactory.getLog(DisworkFileSystem.class); + + /** storage will permit to save and read directories, files and links */ + protected Storage storage; + + /** number of try to acquire a lock before giving up */ + protected static final int LOCK_MAX_NUMBER_OF_TRY = 10; + + /** time (ms) to wait between two try to acquire a lock */ + protected static final int LOCK_WAIT = 10 * 1000; // ten seconds + + /** default constructor (uses default configuration parameters) + * @throws IOException caused by network issue + */ + public DisworkFileSystem() throws IOException { + this(new DisworkFileSystemConfig()); + } + + /** constructor allowing to provide another configuration to use + * + * @param config the configuration + * @throws IOException caused by network issue + */ + public DisworkFileSystem(DisworkFileSystemConfig config) + throws IOException { + storage = new Storage(config); + } + + /** tests the existence of a file/dir/link at a given path + * return true if something exists at path p, it will be true if a call + * to <code>mkdir(p)</code>, <code>write(p, ?)</code> or + * <code>ln(p, ?)</code> has been done before. + * @param path a path in the virtual FS + * @return true is something (a link, a file, or a directory) exists at path + * @throws IOException + */ + public boolean exists(String path) throws IOException { + String entry = walk(path); + boolean result = entry != null; + return result; + } + + /** + * use this method to read the content of a file. A call of read may be + * done after a call to {@link #write} + * @param path the path to the file to read + * @return an InputStream on the file + * @throws FileNotFoundException if no file exists at this path + * @throws IOException if path exists but is a directory + */ + public InputStream read(String path) throws FileNotFoundException, + IOException { + + String entry = walk(path); + if (entry == null) { + throw new FileNotFoundException(path); + } + + InputStream result = null; + + if (EntryUtil.isLink(entry)) { + log.info("reading link " + path); + String id = EntryUtil.getIdFromEntry(entry); + String link = storage.getLink(id); + String newTarget = EntryUtil.resolveLink(path, link); + result = read(newTarget); + } else if (EntryUtil.isDirectory(entry)) { + throw new IOException("target is not a file: " + path); + } else if (EntryUtil.isFile(entry)) { + log.info("reading file " + path); + String id = EntryUtil.getIdFromEntry(entry); + result = storage.getFile(id); + } + + log.info("read " + path + " returns " + result.available() + " bytes"); + + return result; + } + + /** + * write a file on the file system at a given place + * @param path + * @param source + * @throws IOException if file already exists + * @throws ConcurrentModificationException if file is already being written + */ + public void write(String path, InputStream source) + throws IOException, + ConcurrentModificationException { + if (source == null) { + throw new IOException("source is not readable (null stream)"); + } + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + log.info("writing " + source.available() + " bytes at " + path); + write(parent, name, source); + } + + protected void write(String parent, String fileName, InputStream source) + throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); + + // checking that file do not already exists in this directory + String content = storage.getDirectory(parentId); + String findResult = EntryUtil.findEntryInDirectory + (content, fileName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + fileName); + } + + // file do not exists, write file on the FS + String newFileId = EntryUtil.generateId(); + storage.putFile(newFileId, source); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // parent dir is locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + write(newTarget, fileName, source); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + } + + /** + * creates a new empty directory + * @param path the absolute path (ending by the name) of the directory + * @throws IOException if parent path is not correct + */ + public void createDirectory(String path) + throws IOException, + ConcurrentModificationException { + String parent = EntryUtil.getParentFromPath(path); + String dirName = EntryUtil.getNameFromPath(path); + createDirectory(parent, dirName); + } + + protected void createDirectory(String parent, String dirName) + throws IOException { + + log.info("trying to create directory " + dirName + " in " + parent); + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); + String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, dirName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + dirName); + } + + // store file before meta info + String newDirectoryId = EntryUtil.generateId(); + storage.putDirectory(newDirectoryId, + EntryUtil.EMPTY_DIRECTORY_CONTENT); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createDirectory(newTarget, dirName); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + } + + /** + * create a new symbolic link + * @param path the path (ending by the name) where the link will be created + * @param target the path where the link point to + * (may be relative or absolute) + * @throws IOException if parent path is not correct + */ + public void createSymbolicLink(String path, String target) + throws IOException, + ConcurrentModificationException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + createSymbolicLink(parent, name, target); + } + + /** + * @see #createSymbolicLink(String, String) + */ + protected void createSymbolicLink(String parent, String name, String target) + throws IOException { + + if (target.startsWith(EntryUtil.PATH_SEPARATOR)) { + // target is absolute + } else { + // target is relative, taking this in consideration + target = EntryUtil.resolveLink(parent, target); + } + + log.info("trying to create symbolic link named " + name + " at path " + + parent + " with target \"" + target + "\""); + + if (exists(target)) { + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + + String parentId = EntryUtil.getIdFromEntry(entryParent); + String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, name); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + name); + } + + // store file before meta info + String newLinkId = EntryUtil.generateId(); + storage.putLink(newLinkId, target); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.L, name, newLinkId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createSymbolicLink(newTarget, name, target); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + + + } else { + throw new IOException(target + " is not a valid target"); + } + } + + /** + * remove a file, directory, or link. Non-empty directories can't be + * removed + * @param path the complete path to the entity to remove + * @throws IOException if path is incorrect or directory not empty + */ + public void delete(String path) throws IOException, + ConcurrentModificationException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + log.info("trying to remove " + path); + delete(parent, name); + } + + /** + * @see #delete(String) + */ + protected void delete(String parent, String name) throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); + String content = storage.getDirectory(parentId); + + String entry = EntryUtil.findEntryInDirectory(content, name); + String idToRemove = EntryUtil.getIdFromEntry(entry); + + // according to the type of the entry name, delete it + if (EntryUtil.isDirectory(entry)) { + // check if not removing a non-empty directory + String innerDirectoryId = EntryUtil.getIdFromEntry(entry); + String innerDirectoryContent = + storage.getDirectory(innerDirectoryId); + // checking the emptiness of the directory + if (!innerDirectoryContent.equals( + EntryUtil.EMPTY_DIRECTORY_CONTENT)) { + // directory is not empty + throw new IOException + ("trying to remove a non-empty directory"); + } + + // remove it + storage.removeDirectory(idToRemove); + + } else if (EntryUtil.isFile(entry)) { + storage.removeFile(idToRemove); + } else if (EntryUtil.isLink(entry)) { + storage.removeLink(idToRemove); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.removeEntryFromEntries(content, name); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + delete(newTarget, name); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + } + + /** + * list the content of a directory + * @param path the complete path to the directory + * @return a list of the names of the elements in <code>path</code> + * @throws IOException if path doesn't point to a directory + */ + public List<String> readDirectory(String path) throws IOException { + String entry = walk(path); + List<String> result = null; + + if (entry == null) { + throw new IOException(path + " doesn't exists"); + } else { + // path may be a link, if it's the case, + // entry become the actual directory + if (EntryUtil.isLink(entry)) { + String target = storage.getLink( + EntryUtil.getIdFromEntry(entry)); + return readDirectory(target); + } else if (EntryUtil.isDirectory(entry)) { + result = new ArrayList<String>(); + String content = storage.getDirectory( + EntryUtil.getIdFromEntry(entry)); + if (EntryUtil.EMPTY_DIRECTORY_CONTENT.equals(content)) { + // directory is empty, add nothing + } else { + String[] entries = content.split( + EntryUtil.ENTRIES_SEPARATOR); + for (String elementEntry : entries) { + result.add(EntryUtil.getNameFromEntry(elementEntry)); + } + } + } else if (EntryUtil.isFile(entry)) { + throw new IOException(path + " is not a directory but a file"); + } else { + log.warn("strange entry" + entry); + throw new IOException("strange entry" + entry); + } + } + + log.info("readDirectory " + path + " returns " + result.size() + " results"); + + return result; + } + + /** + * return the entry of the element at the end of <code>path</code> + * @param path + * @return null if path is not valid + */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + /** + * This method is a recursive function to walk through the tree structure + * of the directories and starting for root directory, following the + * symbolic links to reach the given path + * @param path + * @param current + * @param content + * @return null if path is not valid or the entry corresponding to path + * @throws IOException + */ + protected String walk(String path, String current, String content) + throws IOException { + // FIXME 20105021 bleny works fine but is not understandable + String result = null; + + // if path is "/", recursion can be initiated with this value : + // (it returns "/" as the id where to get the content of "/" + if (path.equals(EntryUtil.ROOT_DIRECTORY)) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY; + } + + String parentPath = EntryUtil.getParentFromPath(path); + + if (content == null) { + // start the recursion from root directory + content = storage.getRootDirectory(); + result = walk(path, EntryUtil.ROOT_DIRECTORY, content); + } else if (parentPath.equals(current)) { + // we are now in the last directory + // ie if path is a/b/p we are at a/b + + // p is the name of the element in a/b we search for + String tail = path.substring(current.length()); + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + + String entry = EntryUtil.findEntryInDirectory(content, p); + result = entry; + } else { + // in middle of path + // if path is a/b/c/d/e/f, we may be at b, c, d... + + String tail; // the path still remaining when in current + // ie if we are in c, tail is "d/e/f" + + // if we are at root directory, deal with the "/" + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + tail = path.substring(current.length()); + } else { + tail = path.substring(current.length() + 1); + } + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); + String p = elementsNames[0]; + + log.info("in intermediate dir " + current + ", looking for " + p); + + // updating current for recursion + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + current = ""; // avoid "//path" next line + } + current += EntryUtil.PATH_SEPARATOR + p; + + String entry = EntryUtil.findEntryInDirectory(content, p); + if (entry == null) { + result = null; + } else { + // we have found the entry to call recursion + + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + String linkContent = storage.getLink(id); + String newTarget = + EntryUtil.resolveLink(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else if (EntryUtil.isFile(entry)) { + // error, found file in path like '/dir1/dir2/filename/dir3' + result = null; + } else { + log.warn("strange case: " + entry); + result = null; + } + } + } + return result; + } + + @Override + public void close() throws IOException { + storage.close(); + } + + protected void setMap(DisworkMap map) { + storage.setMap(map); + } + +} Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,190 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.util.ApplicationConfig; + +/** + * This object contains parameters used in {@link DisworkFileSystem}. + * + * Available parameters are: + * <dl> + * <dt>blocks_size</dt> + * <dd> + * The size of blocks to create when splitting big data in bytes + * (by default, 10 MiB) + * </dd> + * </dl> + * + * + * This class provides utilities to write tests easily. + * + * You can get multiples diswork configs ready to use. All the instances uses + * a new port and the local IP. + * <pre> + * c = newDisworkConfig(); // create a config for a bootstrap node + * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that + * // will bootstrap by joining the + * // first node + * c3 = newDisworkConfig(c.getUsedPort()) + * </pre> + * + */ +public class DisworkFileSystemConfig extends ApplicationConfig { + + private static final Log log = + LogFactory.getLog(DisworkFileSystemConfig.class); + + protected static Integer port = 19000; + + /** + * returns a new port, returned value change at each call. + * @return + */ + public static Integer getPort() { + port += 1; + return port; + } + + /** + * returns the IP on the local machine. Trying to get an public IP or a LAN + * IP or the loopback IP if there is no other interface + * @return + * @throws UnknownHostException + */ + public static String getIp() throws UnknownHostException { + InetAddress result = InetAddress.getLocalHost(); + if (result.isLoopbackAddress()) { + try { + Socket temp = new Socket("microsoft.com", 80); + result = temp.getLocalAddress(); + temp.close(); + } catch (IOException e) { + log.warn("can't get external IP address"); + } + } + return result.getHostAddress(); + } + + public DisworkFileSystemConfig() { + setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB + + setDefaultOption("diswork.fs.map_type", "inmemory"); + setDefaultOption("diswork.fs.use_port", port.toString()); + } + + public int getBlockSize() { + return getOptionAsInt("diswork.fs.blocks_size"); + } + + public Integer getUsedPort() { + return getOptionAsInt("diswork.fs.use_port"); + } + + public void setUsedPort(Integer port) { + setOption("diswork.fs.use_port", port.toString()); + } + + public String getBootstrapIp() { + return getOption("diswork.fs.bootstrap.ip"); + } + + public void setBootstrapIp(String ip) { + setOption("diswork.fs.bootstrap.ip", ip); + } + + public Integer getBootstrapPort() { + return getOptionAsInt("diswork.fs.bootstrap.port"); + } + + public void setBootstrapPort(Integer port) { + setOption("diswork.fs.bootstrap.port", port.toString()); + } + + public String getMapType() { + return getOption("diswork.fs.map_type"); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a single-node instance of DisworkFS + * @return + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig() + throws UnknownHostException { + return newPastryDisworkConfig(null); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a multiple-node instance of DisworkFS on a same machine. + * @param bootstrapPort the port on the same machine where another node can + * be found to bootstrap + * @return a complete config + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "pastry"); + result.setOption("diswork.fs.use_port", port); + result.setOption("diswork.fs.bootstrap.ip", ip); + if (bootstrapPort == null) { + result.setOption("diswork.fs.bootstrap.port", port); + } else { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + } + return result; + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig() + throws UnknownHostException { + return newKademliaDisworkConfig(null); + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "kademlia"); + result.setOption("diswork.fs.use_port", port); + if (bootstrapPort != null) { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + result.setOption("diswork.fs.bootstrap.ip", ip); + } + return result; + } +} Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,35 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +/** + * DisworkFS is a distributed file system. You can use it by instantiating + * {@link org.nuiton.diswork.fs.DisworkFileSystem} and use it to store + * directories and their content. + * + * You can change the default DisworkFileSystem behavior by provide a + * {@link org.nuiton.diswork.fs.DisworkFileSystemConfig} instance at + * construction. If you don't provide one, a instance will be created + */ + +package org.nuiton.diswork.fs; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/DisworkFileSystemTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.peerunit; +package org.nuiton.diswork.fs.peerunit; import static fr.inria.peerunit.test.assertion.Assert.assertTrue; import static fr.inria.peerunit.test.assertion.Assert.assertEquals; @@ -36,8 +36,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystem; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; import fr.inria.peerunit.TestCaseImpl; import fr.inria.peerunit.parser.TestStep; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/package-info.java 2010-06-08 13:16:35 UTC (rev 68) @@ -26,4 +26,4 @@ * In this package can be found some Test classes for PeerUnit framework */ -package org.nuiton.disworkfs.peerunit; \ No newline at end of file +package org.nuiton.diswork.fs.peerunit; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/DisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.Closeable; import java.util.Map; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/EntryUtil.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.nio.charset.Charset; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.IOException; import java.util.Arrays; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.DataInput; import java.io.DataInputStream; @@ -37,7 +37,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.planx.xmlstore.routing.Identifier; import org.planx.xmlstore.routing.Kademlia; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.IOException; import java.net.BindException; @@ -36,7 +36,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; import rice.Continuation; import rice.environment.Environment; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.ByteArrayInputStream; import java.io.Closeable; @@ -36,7 +36,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; /** * This class is the middle layer between the File System operations and Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/package-info.java 2010-06-08 13:16:35 UTC (rev 68) @@ -26,7 +26,7 @@ * * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a * way to persistently store data. This is done by the - * {@link org.nuiton.disworkfs.storage.Storage} class + * {@link org.nuiton.diswork.fs.storage.Storage} class * which permit to store different type of data. * * @@ -108,13 +108,13 @@ * Link), the name of the element, and an ID to be used as * a key on the map to get the actual content. Those three informations are * separated by ":" which is - * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRY_SEPARATOR}. + * {@link org.nuiton.diswork.fs.storage.EntryUtil#ENTRY_SEPARATOR}. * * * * A directory way have multiple entries. Entries of a directory are * separated by "\n" which is - * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRIES_SEPARATOR}. + * {@link org.nuiton.diswork.fs.storage.EntryUtil#ENTRIES_SEPARATOR}. * * * The above description shows the main principle used to store a tree @@ -198,7 +198,7 @@ * A metablock is composed of the total size of the file followed by * an ordered lists of IDs of the different blocks composing the file. * Those informations are separated by ";" (see - * {@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). + * {@link org.nuiton.diswork.fs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). * * * In the above example: @@ -222,10 +222,10 @@ * When reading and writing in storage, split is done transparently. When * reading, a Stream is returned: it loads data blocks after blocks * when needed (see - * {@link org.nuiton.disworkfs.storage.Storage.SplitBlocksInputStream}). + * {@link org.nuiton.diswork.fs.storage.Storage.SplitBlocksInputStream}). * When writing, data are split in blocks of a maximum configurable size * (see {@link org.nuiton.disworkfs.DisworkFileSystemConfig}). * */ -package org.nuiton.disworkfs.storage; \ No newline at end of file +package org.nuiton.diswork.fs.storage; \ No newline at end of file Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,361 @@ +package org.nuiton.diswork.fs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ConcurrentModificationException; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.util.FileUtil; + + +public abstract class AbstractDisworkFileSystemTest { + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = + System.getProperty("java.io.tmpdir", ".") // temp directory + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 9999; + + static protected DisworkFileSystem fileSystem; + + /** + * This setUp creates in a temp directory a file of size + * {@link randomFileSize} (fulfilling it with random bytes) + * This file can be found at {@link #randomFilePath} + * At the end of the method {@link #fileSystem} is ready to be used + * @throws Exception + */ + @Before + public void setUp() throws Exception { + // create a temp directory for our test + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + // creating random data for the file + Random random = new Random(); + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + /** + * writing a file at root directory should not raise any exception + * @throws Exception + */ + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + /** + * First, write a file then test if exists return true. + * @throws Exception + */ + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * tests {@link org.nuiton.diswork.fs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte: contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + source.close(); + readResult.close(); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testFailAtWrite() throws Exception { + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + } + + /** + * this use case should raise an exception because writing to a file + * that already exists + */ + @Test(expected = IOException.class) + public void testFailAtDoubleWrite() throws Exception { + InputStream source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + source.close(); + + source = new FileInputStream(randomFilePath); + try { + fileSystem.write("/my_file", source); + } finally { + source.close(); + } + } + + /** + * This test uses mkdir to create dirs and sub-dirs + * @throws Exception + */ + @Test + public void testCreateDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + /** + * Create some folders with mkdir and write files in those directories + * @throws Exception + */ + @Test + public void testWriteInFolder() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", + new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + /** + * create a symbolic link to a file. This test show that we can read the + * file using the link + * @throws Exception + */ + @Test + public void testLinking() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + /** + * Trying to create a link to a wrong target, this sould raise an exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtLinking() throws Exception { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + } + + /** + * Trying to remove files and directories + * @throws Exception + */ + @Test + public void testRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.delete("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.delete("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + /** + * By trying to remove a non-empty directory, this test should raise an + * exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.delete("/my_folder"); + } + + /** + * This tests uses ls + */ + @Test + public void testListDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.createDirectory("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); + + List<String> lsResult = fileSystem.readDirectory("/my_folder"); + + // checking that result contains all the required data + assertTrue(lsResult.contains("my_sub_dir")); + assertTrue(lsResult.contains("my_file")); + assertTrue(lsResult.contains("my_link")); + + // ... and only those + assertEquals(3, lsResult.size()); + + lsResult = fileSystem.readDirectory("/"); + assertEquals(1, lsResult.size()); + } + + @Test + public void testConcurrency() throws Exception { + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + } + + @Test + public void testLinks() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/otherdir"); + + fileSystem.createSymbolicLink("/link", "dir/subdir"); + fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); + List<String> readResult = fileSystem.readDirectory("/dir/subdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subsubdirlink")); + + fileSystem.createDirectory("/link/subsubdirlink/finaldir"); + readResult = fileSystem.readDirectory("/otherdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("finaldir")); + + // tests that delete remove the link itself and not the target + fileSystem.delete("/link/subsubdirlink"); + assertTrue(fileSystem.exists("/otherdir")); + fileSystem.delete("/link"); + assertTrue(fileSystem.exists("/dir/subdir")); + + // testing the use of multiple links at the end of a path + fileSystem.createSymbolicLink("/link1", "/dir"); + fileSystem.createSymbolicLink("/link2", "/link1"); + fileSystem.createSymbolicLink("/link3", "/link2"); + + readResult = fileSystem.readDirectory("/link3"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subdir")); + } +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,21 @@ +package org.nuiton.diswork.fs; + +import org.junit.Before; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkFileSystemInMemoryTest extends AbstractDisworkFileSystemTest { + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); + disworkConfig.setOption("diswork.fs.map_type", "inmemory"); + fileSystem = new DisworkFileSystem(disworkConfig); + } + +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,79 @@ +package org.nuiton.diswork.fs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkFileSystemKademliaTest extends AbstractDisworkFileSystemTest { + + protected Integer bootstrapPort; + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + } + + + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + + +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,77 @@ +package org.nuiton.diswork.fs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkFileSystemPastryTest extends AbstractDisworkFileSystemTest { + + protected Integer bootstrapPort; + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = DisworkFileSystemConfig.newPastryDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + + } + + + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,4 +1,4 @@ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; @@ -11,6 +11,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.nuiton.diswork.fs.storage.DisworkMap; public abstract class AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/EntryUtilTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -15,7 +15,7 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. *##%*/ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertArrayEquals; @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; import org.junit.Test; -import org.nuiton.disworkfs.storage.EntryUtil; +import org.nuiton.diswork.fs.storage.EntryUtil; /** * Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,6 +1,7 @@ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import org.junit.Before; +import org.nuiton.diswork.fs.storage.InMemoryDisworkMap; public class InMemoryDisworkMapTest extends AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,4 +1,4 @@ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -10,7 +10,8 @@ import org.junit.Before; import org.junit.Test; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.storage.KademliaDisworkMap; import org.planx.xmlstore.routing.Identifier; public class KademliaDisworkMapTest extends AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/PastryDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,3 +1,4 @@ +package org.nuiton.diswork.fs.storage; //package org.nuiton.disworkfs.storage; // //import org.junit.Before;

1 0

r67 - in trunk: diswork-daemon src/site/rst/diswork-daemon
by bleny＠users.nuiton.org 08 Jun '10

08 Jun '10

Author: bleny Date: 2010-06-08 15:03:59 +0200 (Tue, 08 Jun 2010) New Revision: 67 Url: http://nuiton.org/repositories/revision/diswork/67 Log: maj specif Modified: trunk/diswork-daemon/ trunk/diswork-daemon/pom.xml trunk/src/site/rst/diswork-daemon/how_it_works.rst Property changes on: trunk/diswork-daemon ___________________________________________________________________ Added: svn:ignore + target Modified: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-daemon/pom.xml 2010-06-08 13:03:59 UTC (rev 67) @@ -34,6 +34,10 @@ <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> + <dependency> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + </dependency>  <dependency> Modified: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 13:03:59 UTC (rev 67) @@ -18,10 +18,19 @@ To create a job, a node should : +#. Check that the application in the good version is available at + /bin/application/application-version.jar + #. Create a directory in home for each of his jobs /home/ID/job-1, /home/ID/job-2... -#. In a job directory, put job.sh, a command line to run. +#. In a job directory, put job.jsdl, a command line to run. This file contains + necessary information like : + + * The name of the job + * The application name, version + * File staging (needed file and where to get them) + * ... (to be completed) #. Declare the job as available by adding a symlink in /var/jobs/TODO with the link named after the current date, the id of the job owner and

1 0

r66 - in trunk: diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage src/site src/site/rst/diswork-daemon src/site/rst/diswork-fs
by bleny＠users.nuiton.org 08 Jun '10

08 Jun '10

Author: bleny Date: 2010-06-08 14:13:22 +0200 (Tue, 08 Jun 2010) New Revision: 66 Url: http://nuiton.org/repositories/revision/diswork/66 Log: bugfix; documentation; specif de diswork daemon Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/src/site/rst/diswork-daemon/how_it_works.rst trunk/src/site/rst/diswork-fs/history.rst trunk/src/site/rst/diswork-fs/how_it_works.rst trunk/src/site/site_en.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-08 12:13:22 UTC (rev 66) @@ -521,9 +521,8 @@ if (EntryUtil.isLink(entry)) { String target = storage.getLink( EntryUtil.getIdFromEntry(entry)); - entry = walk(target); - } - if (EntryUtil.isDirectory(entry)) { + return readDirectory(target); + } else if (EntryUtil.isDirectory(entry)) { result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); @@ -536,8 +535,11 @@ result.add(EntryUtil.getNameFromEntry(elementEntry)); } } + } else if (EntryUtil.isFile(entry)) { + throw new IOException(path + " is not a directory but a file"); } else { - throw new IOException(path + " is not a directory"); + log.warn("strange entry" + entry); + throw new IOException("strange entry" + entry); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-08 12:13:22 UTC (rev 66) @@ -341,6 +341,20 @@ readResult = fileSystem.readDirectory("/otherdir"); assertEquals(1, readResult.size()); assertTrue(readResult.contains("finaldir")); + + // tests that delete remove the link itself and not the target + fileSystem.delete("/link/subsubdirlink"); + assertTrue(fileSystem.exists("/otherdir")); + fileSystem.delete("/link"); + assertTrue(fileSystem.exists("/dir/subdir")); + + // testing the use of multiple links at the end of a path + fileSystem.createSymbolicLink("/link1", "/dir"); + fileSystem.createSymbolicLink("/link2", "/link1"); + fileSystem.createSymbolicLink("/link3", "/link2"); + readResult = fileSystem.readDirectory("/link3"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subdir")); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) @@ -11,7 +11,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.nuiton.disworkfs.DisworkFileSystemConfig; public abstract class AbstractDisworkMapTest { @@ -21,9 +20,13 @@ protected DisworkMap map1; protected DisworkMap map2; + /** + * should be overridden by method that set map1 and map2 with different + * implementation of DisworkMap + * @throws Exception + */ @Before - public void setUp() throws Exception { - + public void setUp() throws Exception { } @After @@ -40,7 +43,7 @@ * there is no side effect */ @Test - public void testPut() { + public void testPutMakeCopy() { byte[] expected = {0x1, 0x2}; map1.put("key", expected); expected[0] = 0xf; @@ -50,11 +53,7 @@ @Test - public void test() throws Exception { - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - - map1 = new KademliaDisworkMap(config); - + public void testPutGet() throws Exception { byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; map1.put("test", newBytes); @@ -65,12 +64,7 @@ @Test public void test2nodes() throws Exception { - DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); - map1 = new KademliaDisworkMap(config1); - DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); - map2 = new KademliaDisworkMap(config2); - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; log.info("trying put test"); Modified: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,4 +1,82 @@ +======================== How Diswork Daemon works ======================== -Diswork Daemon uses Diswork File System. \ No newline at end of file +Overview +-------- + +Diswork Daemon uses Diswork File System. + +Submit a job +------------ + +First, a node, before joining a network generate, for himself, a unique ID. And +then, create a directory "/home/ID". If this is not the first time the node +is run, ID should be reused so jobs can be found back after disconnect/connect. + +In his home directory, a node can put and remove his jobs. + +To create a job, a node should : + +#. Create a directory in home for each of his jobs + /home/ID/job-1, /home/ID/job-2... + +#. In a job directory, put job.sh, a command line to run. + +#. Declare the job as available by adding a symlink in /var/jobs/TODO + with the link named after the current date, the id of the job owner and + the job name. Places possible in /var/jobs/ are : + +#. Check the content of /home/ID/job-i/status + +#. Finally, clean home by deleting the directories of finished jobs + +Perform a job +------------- + +To perform a job, a node should find a job in /var/jobs/, this directory has +6 sub-directories : + +- /var/jobs/TODO + +- /var/jobs/FAILED_1 + +- /var/jobs/FAILED_2 + +- /var/jobs/FAILED_3 + +- /var/jobs/DONE + +TODO, FAILED_1, FAILED_2 all have a sub-directory RUNNING. + +#. Choose a job in one of those directories except DONE and FAILED_3. Use the + data in the link same to know which job should be done first; + +#. Once the job is chosen, move the symlink to the RUNNING subdir where you + found the link. Then, follow the link to get the actual datas needed for + the job; + +#. Run the job; + +#. Now there is two way to proceed : + + - If the job successfully ran (no exception raised and return code is 0) : + + #. write the results in the job directory; + + #. move the link to DONE. + + - If the job has failed (a job taking too much time may be considered as + a failure). move it to according to where the job was found. If + the job was in TODO, move it to FAILED_1 (read "failed once") ; + if the job was in FAILED_1, move it to FAILED_2, if it was it FAILED_2, + move it to FAILED_3. + + In both cases, write what occurred in the job directory in the status.txt + file (append your message, don't remove anything); + +#. Sometimes, while choosing a job, check in the RUNNING directory if any + symlink is not too old (use the name of the link). It may means that a + node crashed while processing this job. Move it to the RUNNING directory + of the next failure level (TODO → FAILED_1 → FAILED_2 → FAILED_3) and try + process it. \ No newline at end of file Modified: trunk/src/site/rst/diswork-fs/history.rst =================================================================== --- trunk/src/site/rst/diswork-fs/history.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-fs/history.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,5 +1,9 @@ +========== +Historique +========== + Premier prototype -================= +----------------- L'objectif était de réaliser un système opportuniste : à chaque fois que des données sont envoyées par un nœud parce qu'un autre nœud les a demandées, Modified: trunk/src/site/rst/diswork-fs/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,6 +1,10 @@ +============================= How Diswork File System works ============================= +Diswork File System uses a map to store every data +-------------------------------------------------- + Diswork File System uses a map to store data persistently. We illustrate the use of a file system by setting the content of "/", @@ -122,4 +126,11 @@ When reading and writing in storage, split is done transparently. When reading, a Stream is returned: it loads data blocks after blocks when needed inner class Storage.SplitBlocksInputStream}). -When writing, data are split in blocks of a maximum configurable size. \ No newline at end of file +When writing, data are split in blocks of a maximum configurable size. + +Diswork File System manage concurrency +-------------------------------------- + +DFS uses copy-on-write to deal with concurrency. Nothing is overridden, when +a file is replaced, the new file is written on the FS without removing the +old file. The old file data is kept and actually removed later. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/site_en.xml 2010-06-08 12:13:22 UTC (rev 66) @@ -13,26 +13,26 @@ </poweredBy> <body> - + <breadcrumbs> <item name="${project.name}" href="index.html" /> </breadcrumbs> - + <menu name="Users"> <item href="user/install.html" name="Install"/> <item href="user/starting.html" name="Starting"/> </menu> - + <menu name="Developer"> <item href="devel/draft.html" name="First draft"/> </menu> - + <menu name="diswork daemon"> - <item href="diswork-fs/how_it_works.html" name="How it works"/> + <item href="diswork-daemon/how_it_works.html" name="How it works"/> </menu> - + <menu name="diswork FS"> - <item href="diswork-fs/history.html" name="Install"/> + <item href="diswork-fs/history.html" name="History of the project"/> <item href="diswork-fs/how_it_works.html" name="How it works"/> </menu>

1 0

r65 - in trunk: . diswork-daemon diswork-daemon/src diswork-daemon/src/main diswork-daemon/src/test diswork-fs/src/main/java/org/nuiton/disworkfs/storage src/site src/site/rst src/site/rst/diswork-daemon
by bleny＠users.nuiton.org 07 Jun '10

07 Jun '10

Author: bleny Date: 2010-06-07 18:24:46 +0200 (Mon, 07 Jun 2010) New Revision: 65 Url: http://nuiton.org/repositories/revision/diswork/65 Log: creation module diswork daemon, code style Added: trunk/diswork-daemon/ trunk/diswork-daemon/LICENSE.txt trunk/diswork-daemon/README.txt trunk/diswork-daemon/changelog.txt trunk/diswork-daemon/pom.xml trunk/diswork-daemon/src/ trunk/diswork-daemon/src/main/ trunk/diswork-daemon/src/main/java/ trunk/diswork-daemon/src/test/ trunk/diswork-daemon/src/test/java/ trunk/src/site/rst/diswork-daemon/ trunk/src/site/rst/diswork-daemon/how_it_works.rst Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/pom.xml trunk/src/site/site_en.xml Added: trunk/diswork-daemon/LICENSE.txt =================================================================== --- trunk/diswork-daemon/LICENSE.txt (rev 0) +++ trunk/diswork-daemon/LICENSE.txt 2010-06-07 16:24:46 UTC (rev 65) @@ -0,0 +1,166 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + Added: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml (rev 0) +++ trunk/diswork-daemon/pom.xml 2010-06-07 16:24:46 UTC (rev 65) @@ -0,0 +1,49 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.nuiton</groupId> + <artifactId>diswork</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <groupId>org.nuiton.diswork</groupId> + <artifactId>diswork-daemon</artifactId> + + <packaging>jar</packaging> + <name>Diswork daemon</name> + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>diswork-fs</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + </dependency> + +  + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + </build> + +</project> \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-07 15:48:17 UTC (rev 64) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-07 16:24:46 UTC (rev 65) @@ -102,8 +102,10 @@ if (blockIdsIndex < blockIds.length) { // closing previously opened stream IOUtils.closeQuietly(currentBlock); - currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); - log.debug("new current block (size = " + currentBlock.available() + ")"); + currentBlock = new ByteArrayInputStream( + map.get(blockIds[blockIdsIndex])); + log.debug("new current block (size = " + + currentBlock.available() + ")"); } else { return -1; } @@ -128,10 +130,11 @@ String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; - numberOfAvailableBytes = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + numberOfAvailableBytes = + EntryUtil.getTotalSizeFromMetaBlock(metaBlock); log.debug("initializing stream with meta block \"" + metaBlock - + "\" (" + blockIds.length + " blocks, " + numberOfAvailableBytes - + " bytes)"); + + "\" (" + blockIds.length + " blocks, " + + numberOfAvailableBytes + " bytes)"); } } @@ -335,7 +338,8 @@ unLock(key); } else { - throw new ConcurrentModificationException("key " + key + " is locked"); + throw new ConcurrentModificationException("key " + key + + " is locked"); } } @@ -385,7 +389,9 @@ Boolean result = null; // if there was no lock or if current lock is mine - if (lock == null || (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { + if (lock == null || + (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { + // file is not locked, we have took the lock result = true; } else { @@ -430,7 +436,8 @@ if (obsoleteMetaBlock != null) { String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); - log.info("removing " + obsoleteBlocksIds.length + " old blocks"); + log.info("removing " + obsoleteBlocksIds.length + + " old blocks"); for (String obsoleteBlockId : obsoleteBlocksIds) { removeKey(obsoleteBlockId); } Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-07 15:48:17 UTC (rev 64) +++ trunk/pom.xml 2010-06-07 16:24:46 UTC (rev 65) @@ -19,6 +19,7 @@ <modules> <module>diswork-fs</module> + <module>diswork-daemon</module> </modules> <dependencyManagement> Added: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst (rev 0) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) @@ -0,0 +1,4 @@ +How Diswork Daemon works +======================== + +Diswork Daemon uses Diswork File System. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-07 15:48:17 UTC (rev 64) +++ trunk/src/site/site_en.xml 2010-06-07 16:24:46 UTC (rev 65) @@ -27,6 +27,10 @@ <item href="devel/draft.html" name="First draft"/> </menu> + <menu name="diswork daemon"> + <item href="diswork-fs/how_it_works.html" name="How it works"/> + </menu> + <menu name="diswork FS"> <item href="diswork-fs/history.html" name="Install"/> <item href="diswork-fs/how_it_works.html" name="How it works"/>

1 0

r64 - in trunk: . diswork-fs diswork-fs/src/main/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 07 Jun '10

07 Jun '10

Author: bleny Date: 2010-06-07 17:48:17 +0200 (Mon, 07 Jun 2010) New Revision: 64 Url: http://nuiton.org/repositories/revision/diswork/64 Log: maj pom Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-07 14:19:26 UTC (rev 63) +++ trunk/diswork-fs/pom.xml 2010-06-07 15:48:17 UTC (rev 64) @@ -38,11 +38,11 @@ <artifactId>PeerUnit</artifactId> </dependency> <dependency> - <groupId>org.planx.xmlstore</groupId> - <artifactId>xmlstore</artifactId> + <groupId>org.planx</groupId> + <artifactId>koala-xmlstore</artifactId> </dependency> - +  <dependency> <groupId>junit</groupId> @@ -50,7 +50,7 @@ <scope>test</scope> </dependency> </dependencies> - + <build> <plugins> <plugin> @@ -65,5 +65,5 @@ </plugin> </plugins> </build> - + </project> \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-07 14:19:26 UTC (rev 63) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-07 15:48:17 UTC (rev 64) @@ -29,6 +29,8 @@ import java.net.Socket; import java.net.UnknownHostException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; /** @@ -59,6 +61,9 @@ */ public class DisworkFileSystemConfig extends ApplicationConfig { + private static final Log log = + LogFactory.getLog(DisworkFileSystemConfig.class); + protected static Integer port = 19000; /** @@ -84,7 +89,7 @@ result = temp.getLocalAddress(); temp.close(); } catch (IOException e) { - // TODO 20100602 bleny do something ? + log.warn("can't get external IP address"); } } return result.getHostAddress(); @@ -100,11 +105,6 @@ public int getBlockSize() { return getOptionAsInt("diswork.fs.blocks_size"); } - - @Deprecated - public boolean useInMemoryMap() { - return "inmemory".equals(getOption("diswork.fs.map_type")); - } public Integer getUsedPort() { return getOptionAsInt("diswork.fs.use_port"); Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-07 14:19:26 UTC (rev 63) +++ trunk/pom.xml 2010-06-07 15:48:17 UTC (rev 64) @@ -79,8 +79,8 @@ <version>2.1</version> </dependency> <dependency> - <groupId>org.planx.xmlstore</groupId> - <artifactId>xmlstore</artifactId> + <groupId>org.planx</groupId> + <artifactId>koala-xmlstore</artifactId> <version>0.4.12</version> </dependency> <dependency>

1 0

r63 - in trunk: . diswork-fs diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit diswork-fs/src/main/java/org/nuiton/disworkfs/storage diswork-fs/src/main/resources diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org 07 Jun '10

07 Jun '10

Author: bleny Date: 2010-06-07 16:19:26 +0200 (Mon, 07 Jun 2010) New Revision: 63 Url: http://nuiton.org/repositories/revision/diswork/63 Log: doc, commentaires, headers, pom (negligeance des tests impliquant Pastry) Added: trunk/diswork-fs/run_demo.sh trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java trunk/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/pom.xml 2010-06-07 14:19:26 UTC (rev 63) @@ -50,4 +50,20 @@ <scope>test</scope> </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> +  + <excludes> + <exclude>**/*Pastry*.java</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> + </project> \ No newline at end of file Added: trunk/diswork-fs/run_demo.sh =================================================================== --- trunk/diswork-fs/run_demo.sh (rev 0) +++ trunk/diswork-fs/run_demo.sh 2010-06-07 14:19:26 UTC (rev 63) @@ -0,0 +1,2 @@ +mvn -e exec:java -Dexec.mainClass=org.nuiton.disworkfs.Demo -Dexec.args="consumer 9001" -Dexec.classpathScope=test +mvn -e exec:java -Dexec.mainClass=org.nuiton.disworkfs.Demo -Dexec.args="producer 9002 127.0.0.1 9001" -Dexec.classpathScope=test Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-07 14:19:26 UTC (rev 63) @@ -37,6 +37,11 @@ private static final Log log = LogFactory.getLog(Demo.class); + protected static final String USAGE = + "Usage :\n" + + "org.nuiton.disworkfs.Demo producer|consumer" + + " usePort [bootStrapIp bootStrapPort]"; + protected static DisworkFileSystem fileSystem; protected static class Producer implements Runnable { @@ -47,66 +52,42 @@ @Override public void run() { while (true) { - try { if (fileSystem.exists("/todo")) { - if (waitingForJob == null) { - Random random = new Random(); - waitingForJob = "/todo/job-" + random.nextInt(); Integer randomInteger = random.nextInt(); expectedResult = randomInteger + 1; String job = randomInteger.toString(); - InputStream source = IOUtils.toInputStream(job); - - - log.info("writing job " + waitingForJob + " (" + randomInteger + ")"); - + log.info("writing job " + waitingForJob + " (" + + randomInteger + ")"); fileSystem.write(waitingForJob, source); - source.close(); - - List<String> todos = fileSystem.readDirectory("/todo"); - - System.out.println("number of jobs : " + todos.size()); - } else { String resultPath = waitingForJob + ".result"; if (fileSystem.exists(resultPath)) { log.info("results has been published"); - - InputStream resultStream = fileSystem.read(resultPath); - - String resultString = IOUtils.toString(resultStream); - + InputStream resultStream = + fileSystem.read(resultPath); + String resultString = + IOUtils.toString(resultStream); Integer result = Integer.parseInt(resultString); - - log.info("result is " + result + "(expected : " + expectedResult + ")"); - - System.out.println("result is " + result + "(expected : " + expectedResult + ")"); - + log.info("result is " + result + " (expected : " + + expectedResult + ")"); fileSystem.delete(resultPath); fileSystem.delete(waitingForJob); - waitingForJob = null; expectedResult = null; - } - } - - } else { fileSystem.createDirectory("/todo"); } - } catch (IOException e) { log.error(e); } - try { Thread.sleep(15 * 1000); } catch (InterruptedException e) { @@ -121,90 +102,86 @@ @Override public void run() { while (true) { - try { if (fileSystem.exists("/todo")) { - List<String> todos = fileSystem.readDirectory("/todo"); - - System.out.println("todos " + todos.size() + " jobs"); - - if (todos.isEmpty()) { log.info("nothing to do"); } else { // taking a random job Random random = new Random(); - String todo = todos.get(random.nextInt(todos.size())); - + String todo = todos.get( + random.nextInt(todos.size()) + ); if (!todo.endsWith(".result")) { String jobPath = "/todo/" + todo; log.info("reading the job " + jobPath); - InputStream in = fileSystem.read(jobPath); - String operation = IOUtils.toString(in); - // in.close(); - log.info("operation to do " + operation); - Integer i = Integer.parseInt(operation); - i += 1; - String result = i.toString(); - log.info("result is " + result); - - InputStream source = IOUtils.toInputStream(result); - + InputStream source = + IOUtils.toInputStream(result); fileSystem.write(jobPath + ".result", source); - source.close(); - } } - } else { fileSystem.createDirectory("/todo"); } - } catch (IOException e) { log.error(e); } - try { Thread.sleep(10 * 1000); } catch (InterruptedException e) { log.info("sleep interrupted", e); } - } } } public static void printUsage() { - System.out.println( - "Usage :\n" - + "org.nuiton.disworkfs.Demo producer|consumer usePort [bootStrapIp bootStrapPort]" - ); + System.out.println(USAGE); } /** + * + * <dl> + * <dt>producer|consumer (required)</dt> + * <dd>a producer will produce operations todo, a consumer may read + * operations and try to execute them + * </dd> + * <dt>port (required)</dt> + * <dd>the port to use</dd> + * <dt>bootstrapip (optional)</dt> + * <dd>if not provided, the node will bootstrap itself, creating a + * new network. If provided, the node will first try to join a + * network</dd> + * <dt>bootstrap port (required if bootstrapip is set)</dt> + * <dd>the port used by the node to join</dd> + * </dl> + * + * Example : + * org.nuiton.disworkfs.Demo consumer 9001 + * org.nuiton.disworkfs.Demo producer 9002 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9003 127.0.0.1 9002 + * org.nuiton.disworkfs.Demo producer 9004 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9005 127.0.0.1 9003 + * * @param args * @throws IOException */ public static void main(String[] args) throws IOException { - if (args.length == 2) { - - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); - fileSystem = new DisworkFileSystem(config); - if ("producer".equals(args[0])) { Thread t = new Thread(new Producer()); log.info("starting a producer"); @@ -214,17 +191,13 @@ log.info("starting a consumer"); t.start(); } - } else if (args.length == 4) { - - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); config.setOption("diswork.fs.bootstrap.ip", args[2]); config.setOption("diswork.fs.bootstrap.port", args[3]); - fileSystem = new DisworkFileSystem(config); - if ("producer".equals(args[0])) { Thread t = new Thread(new Producer()); log.info("starting a producer"); @@ -234,10 +207,8 @@ log.info("starting a consumer"); t.start(); } - } else { printUsage(); } } - -} +} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-07 14:19:26 UTC (rev 63) @@ -572,29 +572,26 @@ // FIXME 20105021 bleny works fine but is not understandable String result = null; + // if path is "/", recursion can be initiated with this value : + // (it returns "/" as the id where to get the content of "/" if (path.equals(EntryUtil.ROOT_DIRECTORY)) { return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + EntryUtil.ROOT_DIRECTORY; } - + String parentPath = EntryUtil.getParentFromPath(path); - + if (content == null) { - // start of walk + // start the recursion from root directory content = storage.getRootDirectory(); result = walk(path, EntryUtil.ROOT_DIRECTORY, content); } else if (parentPath.equals(current)) { - // in the last directory - // recuperation de l'element a traiter dans p + // we are now in the last directory + // ie if path is a/b/p we are at a/b + // p is the name of the element in a/b we search for String tail = path.substring(current.length()); - - /* - log.info("tail " + tail); - String[] elementsNames = tail.split("/"); - String p = elementsNames[0]; - */ String p = EntryUtil.getNameFromPath(tail); log.info("in final dir " + current + ", looking for " + p); @@ -603,9 +600,12 @@ result = entry; } else { // in middle of path - + // if path is a/b/c/d/e/f, we may be at b, c, d... + String tail; // the path still remaining when in current + // ie if we are in c, tail is "d/e/f" + // if we are at root directory, deal with the "/" if (current.equals(EntryUtil.ROOT_DIRECTORY)) { tail = path.substring(current.length()); } else { @@ -619,7 +619,7 @@ log.info("in intermediate dir " + current + ", looking for " + p); - // mise a jour de current + // updating current for recursion if (current.equals(EntryUtil.ROOT_DIRECTORY)) { current = ""; // avoid "//path" next line } @@ -629,6 +629,8 @@ if (entry == null) { result = null; } else { + // we have found the entry to call recursion + if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getIdFromEntry(entry); content = storage.getDirectory(id); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-07 14:19:26 UTC (rev 63) @@ -94,11 +94,7 @@ setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB setDefaultOption("diswork.fs.map_type", "inmemory"); - setDefaultOption("diswork.fs.use_port", "9001"); - /* - setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); - setDefaultOption("diswork.fs.bootstrap.port", "9001"); - */ + setDefaultOption("diswork.fs.use_port", port.toString()); } public int getBlockSize() { Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java 2010-06-07 14:19:26 UTC (rev 63) @@ -0,0 +1,29 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +/** + * In this package can be found some Test classes for PeerUnit framework + */ + +package org.nuiton.disworkfs.peerunit; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-07 14:19:26 UTC (rev 63) @@ -27,4 +27,4 @@ import java.io.Closeable; import java.util.Map; -public interface DisworkMap extends Map<String, byte[]>, Closeable {} +public interface DisworkMap extends Map<String, byte[]>, Closeable {} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-07 14:19:26 UTC (rev 63) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.DataInput; @@ -4,7 +28,6 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import java.math.BigInteger; import java.net.InetSocketAddress; import java.util.Collection; import java.util.Map; @@ -17,14 +40,13 @@ import org.nuiton.disworkfs.DisworkFileSystemConfig; import org.planx.xmlstore.routing.Identifier; import org.planx.xmlstore.routing.Kademlia; -import org.planx.xmlstore.routing.RoutingException; public class KademliaDisworkMap implements DisworkMap { private static final Log log = LogFactory.getLog(KademliaDisworkMap.class); protected Kademlia kad; - + public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); @@ -43,8 +65,6 @@ UUID uuid = UUID.nameUUIDFromBytes(EntryUtil.stringToBytes(s)); s = uuid.toString(); - - /* byte[] bytes = EntryUtil.stringToBytes(s); Identifier id = null; @@ -108,7 +128,6 @@ byte[] previousValue = null; try { previousValue = get(key); - Identifier id = stringToIdentifier((String) key); kad.put(id, value); } catch (Exception e) { @@ -124,7 +143,6 @@ byte[] previousValue = null; try { previousValue = get(key); - Identifier id = stringToIdentifier((String) key); kad.remove(id); } catch (Exception e) { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-07 14:19:26 UTC (rev 63) @@ -80,7 +80,8 @@ } } - protected abstract class MyContinuation<R, E extends Exception> implements Continuation<R, Exception> { + protected abstract class MyContinuation<R, E extends Exception> + implements Continuation<R, Exception> { protected boolean finished = false; @@ -184,7 +185,8 @@ // abort if can't join if (node.joinFailed()) { - throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason()); + throw new IOException("Could not join the FreePastry ring. " + + "Reason : " + node.joinFailedReason()); } } } @@ -216,7 +218,8 @@ } - protected class ContainsKeyContinuation extends MyContinuation<Boolean, Exception> { + protected class ContainsKeyContinuation extends + MyContinuation<Boolean, Exception> { public Boolean result = null; @@ -231,15 +234,16 @@ @Override public boolean containsKey(Object key) { Id id = pastryIdFactory.buildId((String) key); - ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation(); - past.existsInOverlay(id, containsKeyContinuation); - containsKeyContinuation.waitUntilFinished(); - boolean result = containsKeyContinuation.result; + ContainsKeyContinuation continuation = new ContainsKeyContinuation(); + past.existsInOverlay(id, continuation); + continuation.waitUntilFinished(); + boolean result = continuation.result; log.info("containsKey " + key + " (id = " + id + ") returns " + result); return result; } - protected class GetContinuation extends MyContinuation<PastContent, Exception> { + protected class GetContinuation extends + MyContinuation<PastContent, Exception> { public byte[] result = null; @@ -278,7 +282,8 @@ return result; } - protected class PutContinuation extends MyContinuation<Boolean[], Exception> { + protected class PutContinuation extends + MyContinuation<Boolean[], Exception> { @Override public void receiveResult(Boolean[] result) { log.info("insert result received : " + Arrays.toString(result)); @@ -311,7 +316,8 @@ return previousValue; } - protected class RemoveContinuation extends MyContinuation<Boolean, Exception> { + protected class RemoveContinuation extends + MyContinuation<Boolean, Exception> { public Boolean result = null; @@ -334,7 +340,8 @@ RemoveContinuation removeContinuation = new RemoveContinuation(); past.remove(id, removeContinuation); removeContinuation.waitUntilFinished(); - log.info("atomic remove " + key + " has returned " + removeContinuation.result); + log.info("atomic remove " + key + " has returned " + + removeContinuation.result); } @Override @@ -344,7 +351,8 @@ log.info("remove " + key + " returns null"); } else { atomicRemove(key); - log.info("remove " + key + " returns " + previousValue.length + " bytes"); + log.info("remove " + key + " returns " + previousValue.length + + " bytes"); } return previousValue; } Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-07 14:19:26 UTC (rev 63) @@ -5,5 +5,5 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=INFO +log4j.logger.org.nuiton.disworkfs=WARN log4j.logger.org.nuiton.disworkfs.Demo=INFO \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -2,7 +2,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -10,8 +9,6 @@ import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.Before; import org.junit.Test; Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -2,7 +2,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -10,8 +9,6 @@ import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.Before; import org.junit.Test; @@ -32,7 +29,7 @@ } - /* + @Test public void testMultipleNodes1() throws Exception { DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); @@ -61,19 +58,18 @@ assertArrayEquals(bytes, getResult); } - /* + @Test public void testMultipleNodes3() throws Exception { fileSystem.createDirectory("/mydir"); try { - log.info("----------------------trying to create second directory"); fileSystem.createDirectory("/myseconddir"); } catch (ConcurrentModificationException e) { fail(); } } - */ + } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -28,12 +28,10 @@ @After public void tearDown() throws Exception { - /* if (map1 != null) map1.close(); if (map2 != null) map2.close(); - */ } /** Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -1,13 +1,17 @@ package org.nuiton.disworkfs.storage; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.junit.Before; import org.junit.Test; import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.planx.xmlstore.routing.Identifier; public class KademliaDisworkMapTest extends AbstractDisworkMapTest { @@ -20,12 +24,42 @@ map2 = new KademliaDisworkMap(config2); } + /** + * this tests the key generation algorithm, it has different properties to + * check : + * * two call with same param should return same key + * * two call with different parameter should return differents keys + * * a call should never return null + */ @Test public void testStringToIndentifier() { - assertNotNull(KademliaDisworkMap.stringToIdentifier("/")); - assertNotNull(KademliaDisworkMap.stringToIdentifier( - "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" - /* + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" - + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir"*/)); + + List<String> paths = new ArrayList<String>(); + + // a set will remove doubloons + Set<Identifier> keys = new HashSet<Identifier>(); + + paths.add("/"); + paths.add("/dir"); + paths.add("/dir/dir"); + paths.add("/a_directory_with_a_long_name_may_lead_to_the_raise_of_an_" + + "exception"); + paths.add("1111111111111111111111111111111111111111111111111111111111"); + paths.add("1111111111111111111111111111111111111111111111111111111112"); + + for (String path : paths) { + Identifier key = KademliaDisworkMap.stringToIdentifier(path); + assertNotNull(key); + + // keys should be equals (depend only from path, not random factor) + Identifier otherKey = KademliaDisworkMap.stringToIdentifier(path); + assertEquals(key, otherKey); + + keys.add(key); + } + + // check differents paths have different keys + // if keys set is too small, there are collisions + assertEquals(paths.size(), keys.size()); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -1,17 +1,17 @@ -package org.nuiton.disworkfs.storage; - -import org.junit.Before; -import org.nuiton.disworkfs.DisworkFileSystemConfig; - -public class PastryDisworkMapTest extends AbstractDisworkMapTest { - - @Before - public void setUp() throws Exception { - DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newPastryDisworkConfig(); - map1 = new PastryDisworkMap(config1); - - DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newPastryDisworkConfig(config1.getUsedPort()); - map2 = new PastryDisworkMap(config2); - } - -} +//package org.nuiton.disworkfs.storage; +// +//import org.junit.Before; +//import org.nuiton.disworkfs.DisworkFileSystemConfig; +// +//public class PastryDisworkMapTest extends AbstractDisworkMapTest { +// +// @Before +// public void setUp() throws Exception { +// DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newPastryDisworkConfig(); +// map1 = new PastryDisworkMap(config1); +// +// DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newPastryDisworkConfig(config1.getUsedPort()); +// map2 = new PastryDisworkMap(config2); +// } +// +//} Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/pom.xml 2010-06-07 14:19:26 UTC (rev 63) @@ -8,11 +8,11 @@   - <parent> - <groupId>org.nuiton</groupId> - <artifactId>mavenpom4redmine</artifactId> - <version>2.1.5</version> - </parent> + <parent> + <groupId>org.nuiton</groupId> + <artifactId>mavenpom4redmine</artifactId> + <version>2.1.5</version> + </parent> <artifactId>diswork</artifactId> <version>0.0.1-SNAPSHOT</version> @@ -79,11 +79,16 @@ <version>2.1</version> </dependency> <dependency> - <groupId>org.planx.xmlstore</groupId> - <artifactId>xmlstore</artifactId> - <version>0.4.12</version> + <groupId>org.planx.xmlstore</groupId> + <artifactId>xmlstore</artifactId> + <version>0.4.12</version> </dependency> - + <dependency> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + <version>2.0</version> + </dependency> +  <dependency> <groupId>junit</groupId>

1 0