June 2010 - Diswork-commits - mailman.cloud.codelutin.com

r62 - in trunk: . diswork-fs diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit diswork-fs/src/main/java/org/nuiton/disworkfs/storage diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org June 4, 2010

June 4, 2010

Author: bleny Date: 2010-06-04 20:15:37 +0200 (Fri, 04 Jun 2010) New Revision: 62 Url: http://nuiton.org/repositories/revision/diswork/62 Log: implementation kademlia, tests, demo, fabriques de configs, remaniement des tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java Removed: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java trunk/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/pom.xml 2010-06-04 18:15:37 UTC (rev 62) @@ -37,6 +37,11 @@ <groupId>fr.inria.peerunit</groupId> <artifactId>PeerUnit</artifactId> </dependency> + <dependency> + <groupId>org.planx.xmlstore</groupId> + <artifactId>xmlstore</artifactId> + </dependency> +  <dependency> Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-04 18:15:37 UTC (rev 62) @@ -53,7 +53,6 @@ if (waitingForJob == null) { - // taking a random job Random random = new Random(); waitingForJob = "/todo/job-" + random.nextInt(); @@ -87,6 +86,8 @@ log.info("result is " + result + "(expected : " + expectedResult + ")"); + System.out.println("result is " + result + "(expected : " + expectedResult + ")"); + fileSystem.delete(resultPath); fileSystem.delete(waitingForJob); @@ -126,6 +127,9 @@ List<String> todos = fileSystem.readDirectory("/todo"); + System.out.println("todos " + todos.size() + " jobs"); + + if (todos.isEmpty()) { log.info("nothing to do"); } else { @@ -195,7 +199,7 @@ if (args.length == 2) { - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); @@ -213,7 +217,7 @@ } else if (args.length == 4) { - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); config.setOption("diswork.fs.bootstrap.ip", args[2]); @@ -221,11 +225,11 @@ fileSystem = new DisworkFileSystem(config); - if ("producer".equals(args[2])) { + if ("producer".equals(args[0])) { Thread t = new Thread(new Producer()); log.info("starting a producer"); t.start(); - } else if ("consumer".equals(args[2])) { + } else if ("consumer".equals(args[0])) { Thread t = new Thread(new Consumer()); log.info("starting a consumer"); t.start(); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-04 18:15:37 UTC (rev 62) @@ -34,6 +34,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.storage.DisworkMap; import org.nuiton.disworkfs.storage.EntryUtil; import org.nuiton.disworkfs.storage.Storage; @@ -657,5 +658,9 @@ public void close() throws IOException { storage.close(); } + + protected void setMap(DisworkMap map) { + storage.setMap(map); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-04 18:15:37 UTC (rev 62) @@ -24,6 +24,11 @@ */ package org.nuiton.disworkfs; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + import org.nuiton.util.ApplicationConfig; /** @@ -37,34 +42,153 @@ * (by default, 10 MiB) * </dd> * </dl> + * + * + * This class provides utilities to write tests easily. + * + * You can get multiples diswork configs ready to use. All the instances uses + * a new port and the local IP. + * <pre> + * c = newDisworkConfig(); // create a config for a bootstrap node + * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that + * // will bootstrap by joining the + * // first node + * c3 = newDisworkConfig(c.getUsedPort()) + * </pre> + * */ public class DisworkFileSystemConfig extends ApplicationConfig { + protected static Integer port = 19000; + + /** + * returns a new port, returned value change at each call. + * @return + */ + public static Integer getPort() { + port += 1; + return port; + } + + /** + * returns the IP on the local machine. Trying to get an public IP or a LAN + * IP or the loopback IP if there is no other interface + * @return + * @throws UnknownHostException + */ + public static String getIp() throws UnknownHostException { + InetAddress result = InetAddress.getLocalHost(); + if (result.isLoopbackAddress()) { + try { + Socket temp = new Socket("microsoft.com", 80); + result = temp.getLocalAddress(); + temp.close(); + } catch (IOException e) { + // TODO 20100602 bleny do something ? + } + } + return result.getHostAddress(); + } + public DisworkFileSystemConfig() { setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB - setDefaultOption("diswork.fs.use_in_memory_map", "false"); + + setDefaultOption("diswork.fs.map_type", "inmemory"); setDefaultOption("diswork.fs.use_port", "9001"); - setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); + /* + setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); setDefaultOption("diswork.fs.bootstrap.port", "9001"); + */ } public int getBlockSize() { return getOptionAsInt("diswork.fs.blocks_size"); } + @Deprecated public boolean useInMemoryMap() { - return getOptionAsBoolean("diswork.fs.use_in_memory_map"); + return "inmemory".equals(getOption("diswork.fs.map_type")); } public Integer getUsedPort() { return getOptionAsInt("diswork.fs.use_port"); } + public void setUsedPort(Integer port) { + setOption("diswork.fs.use_port", port.toString()); + } + public String getBootstrapIp() { return getOption("diswork.fs.bootstrap.ip"); } + public void setBootstrapIp(String ip) { + setOption("diswork.fs.bootstrap.ip", ip); + } + public Integer getBootstrapPort() { return getOptionAsInt("diswork.fs.bootstrap.port"); } + + public void setBootstrapPort(Integer port) { + setOption("diswork.fs.bootstrap.port", port.toString()); + } + + public String getMapType() { + return getOption("diswork.fs.map_type"); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a single-node instance of DisworkFS + * @return + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig() + throws UnknownHostException { + return newPastryDisworkConfig(null); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a multiple-node instance of DisworkFS on a same machine. + * @param bootstrapPort the port on the same machine where another node can + * be found to bootstrap + * @return a complete config + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "pastry"); + result.setOption("diswork.fs.use_port", port); + result.setOption("diswork.fs.bootstrap.ip", ip); + if (bootstrapPort == null) { + result.setOption("diswork.fs.bootstrap.port", port); + } else { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + } + return result; + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig() + throws UnknownHostException { + return newKademliaDisworkConfig(null); + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "kademlia"); + result.setOption("diswork.fs.use_port", port); + if (bootstrapPort != null) { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + result.setOption("diswork.fs.bootstrap.ip", ip); + } + return result; + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -34,6 +34,8 @@ import java.util.Arrays; import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.DisworkFileSystem; import org.nuiton.disworkfs.DisworkFileSystemConfig; @@ -42,28 +44,53 @@ public class DisworkFileSystemTest extends TestCaseImpl { - protected Integer port = 9001; + private static final Log log = + LogFactory.getLog(DisworkFileSystemTest.class); + // protected Integer port = 31000; + protected DisworkFileSystem fileSystem; protected byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; @TestStep(range="0",timeout=1000000, order = 0) public void testConnect() throws Exception { - Integer myPort = port + getId(); - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); - config.setOption("diswork.fs.use_port", myPort.toString()); + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setUsedPort(31000); fileSystem = new DisworkFileSystem(config); + + config.printConfig(); + + log.info("i bootstrap : " + DisworkFileSystemConfig.getIp() + ":" + 31000); + put(0, DisworkFileSystemConfig.getIp()); + put(1, 31000); + } @TestStep(range="1",timeout=1000000, order = 1) public void testConnect1() throws Exception { - Integer myPort = port + getId(); - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); - config.setOption("diswork.fs.use_port", myPort.toString()); + // Integer myPort = port + getId(); + Integer myPort = 31001; + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setUsedPort(myPort); + + // get bootstrap info + + String bootstrapIp = (String) get(0); + Integer bootstrapPort = (Integer) get(1); + + config.setBootstrapIp("127.0.0.1"); + config.setBootstrapPort(bootstrapPort); + + config.printConfig(); + + log.info("bootstrap is " + bootstrapIp + ":" + bootstrapPort); + fileSystem = new DisworkFileSystem(config); } - /* + @TestStep(range="0",timeout=1000000, order = 2) public void testWrite() throws Exception { InputStream source = new ByteArrayInputStream(bytes); @@ -76,34 +103,28 @@ try { InputStream source = fileSystem.read("/myfile"); byte[] readResult = IOUtils.toByteArray(source); - source.close(); + // source.close(); boolean arraysContentEquals = Arrays.equals(bytes, readResult); + + System.out.println("arraysContentEquals = " + arraysContentEquals); assertTrue(arraysContentEquals); } catch (FileNotFoundException e) { + System.out.println("file not found"); fail(); } } - */ + @TestStep(range="1",timeout=1000000, order = 4) public void testCreateDir() throws Exception { fileSystem.createDirectory("/mydir"); - fileSystem.createDirectory("/myseconddir"); } - @TestStep(range="0-1",timeout=1000000, order = 5) - public void fake() throws Exception { - - } - - /* - @TestStep(range="0-1",timeout=1000000, order = 5) + @TestStep(range="0",timeout=1000000, order = 5) public void testCreateDir2() throws Exception { assertTrue(fileSystem.exists("/mydir")); - assertTrue(fileSystem.exists("/myseconddir")); } - */ - /* - @TestStep(range="1",timeout=1000000, order = 6) + + @TestStep(range="0",timeout=1000000, order = 6) public void testCreateSubDir() throws Exception { fileSystem.createDirectory("/mydir/mysubdir"); } @@ -113,5 +134,5 @@ assertTrue(fileSystem.exists("/mydir/mysubdir")); assertEquals(1, fileSystem.readDirectory("/mydir").size()); } - */ + } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,185 @@ +package org.nuiton.disworkfs.storage; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.planx.xmlstore.routing.Identifier; +import org.planx.xmlstore.routing.Kademlia; +import org.planx.xmlstore.routing.RoutingException; + +public class KademliaDisworkMap implements DisworkMap { + + private static final Log log = LogFactory.getLog(KademliaDisworkMap.class); + + protected Kademlia kad; + + public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { + kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); + + if (config.getBootstrapIp() != null) { + InetSocketAddress bootstrap = new InetSocketAddress( + config.getBootstrapIp(), + config.getBootstrapPort()); + kad.connect(bootstrap); + } + + //Identifier.IDSIZE = 1024; + log.info("using " + Identifier.IDSIZE + " bytes for identifiers"); + } + + protected static Identifier stringToIdentifier(String s) { + UUID uuid = UUID.nameUUIDFromBytes(EntryUtil.stringToBytes(s)); + s = uuid.toString(); + + + + /* + byte[] bytes = EntryUtil.stringToBytes(s); + Identifier id = null; + try { + id = new Identifier(bytes); + } catch (java.lang.IndexOutOfBoundsException e) { + log.error("generated id is out of bound : Identifier.IDSIZE " + + " is too small"); + } + */ + + InputStream in = IOUtils.toInputStream(s); + DataInput dataInput = new DataInputStream(in); + Identifier id = null; + try { + id = new Identifier(dataInput); + } catch (IOException e) { + log.error("unable to create key from string " + s); + } + + + log.info("key for string " + s + " is " + id); + return id; + } + + @Override + public void close() throws IOException { + kad.close(); + } + + @Override + public boolean containsKey(Object key) { + Identifier id = stringToIdentifier((String) key); + Boolean result = null; + try { + result = kad.contains(id); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return result; + } + + @Override + public byte[] get(Object key) { + Identifier id = stringToIdentifier((String) key); + byte[] result = null; + try { + result = (byte[]) kad.get(id); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return result; + } + + @Override + public byte[] put(String key, byte[] value) { + byte[] previousValue = null; + try { + previousValue = get(key); + + Identifier id = stringToIdentifier((String) key); + kad.put(id, value); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return previousValue; + } + + @Override + public byte[] remove(Object key) { + byte[] previousValue = null; + try { + previousValue = get(key); + + Identifier id = stringToIdentifier((String) key); + kad.remove(id); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return previousValue; + } + + @Override + @Deprecated + public Collection<byte[]> values() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public void clear() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public boolean containsValue(Object arg0) { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public Set<java.util.Map.Entry<String, byte[]>> entrySet() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public int size() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public void putAll(Map<? extends String, ? extends byte[]> arg0) { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public Set<String> keySet() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public boolean isEmpty() { + throw new UnsupportedOperationException("not yet implemented"); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-04 18:15:37 UTC (rev 62) @@ -147,24 +147,46 @@ protected DisworkFileSystemConfig disworkConfig; - public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { + public Storage(DisworkFileSystemConfig disworkConfig, DisworkMap map) + throws IOException { + this.disworkConfig = disworkConfig; - - // instantiating a map according to config directives - if (disworkConfig.useInMemoryMap()) { - log.info("using in-memory map"); - map = new InMemoryDisworkMap(); + + if (map == null) { + // instantiating a map according to config directives + String mapType = disworkConfig.getOption("diswork.fs.map_type"); + + if (mapType == null) { + log.info("no map type specified"); + this.map = null; + } else { + if ("inmemory".equals(mapType)) { + log.info("using in-memory map"); + this.map = new InMemoryDisworkMap(); + } else if ("pastry".equals(mapType)) { + log.info("using Pastry map"); + this.map = new PastryDisworkMap(disworkConfig); + } else if ("kademlia".equals(mapType)) { + log.info("using Kademlia map"); + this.map = new KademliaDisworkMap(disworkConfig); + } + } } else { - log.info("using Pastry map"); - map = new PastryDisworkMap(disworkConfig); + this.map = map; } + // creating root directory, if needed - if (! map.containsKey(EntryUtil.ROOT_DIRECTORY)) { + if (! this.map.containsKey(EntryUtil.ROOT_DIRECTORY)) { + log.info("creating root directory"); putDirectory(EntryUtil.ROOT_DIRECTORY, EntryUtil.EMPTY_DIRECTORY_CONTENT); } } + + public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { + this(disworkConfig, null); + } /** * @return the content (entries) of the root directory @@ -430,5 +452,9 @@ Long lockAge = currentDate - currentLockTime; return lockAge > LOCK_VALID_TIME; } + + public void setMap(DisworkMap map) { + this.map = map; + } } \ No newline at end of file Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java (from rev 60, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,346 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ConcurrentModificationException; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + + +public abstract class AbstractDisworkFileSystemTest { + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = + System.getProperty("java.io.tmpdir", ".") // temp directory + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 9999; + + static protected DisworkFileSystem fileSystem; + + /** + * This setUp creates in a temp directory a file of size + * {@link randomFileSize} (fulfilling it with random bytes) + * This file can be found at {@link #randomFilePath} + * At the end of the method {@link #fileSystem} is ready to be used + * @throws Exception + */ + @Before + public void setUp() throws Exception { + // create a temp directory for our test + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + // creating random data for the file + Random random = new Random(); + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + /** + * writing a file at root directory should not raise any exception + * @throws Exception + */ + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + /** + * First, write a file then test if exists return true. + * @throws Exception + */ + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte: contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + source.close(); + readResult.close(); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testFailAtWrite() throws Exception { + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + } + + /** + * this use case should raise an exception because writing to a file + * that already exists + */ + @Test(expected = IOException.class) + public void testFailAtDoubleWrite() throws Exception { + InputStream source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + source.close(); + + source = new FileInputStream(randomFilePath); + try { + fileSystem.write("/my_file", source); + } finally { + source.close(); + } + } + + /** + * This test uses mkdir to create dirs and sub-dirs + * @throws Exception + */ + @Test + public void testCreateDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + /** + * Create some folders with mkdir and write files in those directories + * @throws Exception + */ + @Test + public void testWriteInFolder() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", + new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + /** + * create a symbolic link to a file. This test show that we can read the + * file using the link + * @throws Exception + */ + @Test + public void testLinking() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + /** + * Trying to create a link to a wrong target, this sould raise an exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtLinking() throws Exception { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + } + + /** + * Trying to remove files and directories + * @throws Exception + */ + @Test + public void testRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.delete("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.delete("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + /** + * By trying to remove a non-empty directory, this test should raise an + * exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.delete("/my_folder"); + } + + /** + * This tests uses ls + */ + @Test + public void testListDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.createDirectory("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); + + List<String> lsResult = fileSystem.readDirectory("/my_folder"); + + // checking that result contains all the required data + assertTrue(lsResult.contains("my_sub_dir")); + assertTrue(lsResult.contains("my_file")); + assertTrue(lsResult.contains("my_link")); + + // ... and only those + assertEquals(3, lsResult.size()); + + lsResult = fileSystem.readDirectory("/"); + assertEquals(1, lsResult.size()); + } + + @Test + public void testConcurrency() throws Exception { + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + } + + @Test + public void testLinks() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/otherdir"); + + fileSystem.createSymbolicLink("/link", "dir/subdir"); + fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); + List<String> readResult = fileSystem.readDirectory("/dir/subdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subsubdirlink")); + + fileSystem.createDirectory("/link/subsubdirlink/finaldir"); + readResult = fileSystem.readDirectory("/otherdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("finaldir")); + + } +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -2,17 +2,17 @@ import org.junit.Before; -public class DisworkFileSystemInMemoryTest extends DisworkFileSystemTest { +public class DisworkFileSystemInMemoryTest extends AbstractDisworkFileSystemTest { /** - * this code executed after {@link DisworkFileSystemTest#setUp()} + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} * @throws Exception */ @Before public void setUpFileSystem() throws Exception { // finally, initiate the fileSystem DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); - disworkConfig.setOption("diswork.fs.use_in_memory_map", "true"); + disworkConfig.setOption("diswork.fs.map_type", "inmemory"); fileSystem = new DisworkFileSystem(disworkConfig); } Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,80 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Before; +import org.junit.Test; + +public class DisworkFileSystemKademliaTest extends AbstractDisworkFileSystemTest { + + protected Integer bootstrapPort; + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + } + + + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -15,18 +15,18 @@ import org.junit.Before; import org.junit.Test; -public class DisworkFileSystemPastryTest extends DisworkFileSystemTest { +public class DisworkFileSystemPastryTest extends AbstractDisworkFileSystemTest { protected Integer bootstrapPort; /** - * this code executed after {@link DisworkFileSystemTest#setUp()} + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} * @throws Exception */ @Before public void setUpFileSystem() throws Exception { // finally, initiate the fileSystem - DisworkFileSystemConfig disworkConfig1 = Util.newDisworkConfig(); + DisworkFileSystemConfig disworkConfig1 = DisworkFileSystemConfig.newPastryDisworkConfig(); bootstrapPort = disworkConfig1.getUsedPort(); fileSystem = new DisworkFileSystem(disworkConfig1); @@ -35,7 +35,7 @@ /* @Test public void testMultipleNodes1() throws Exception { - DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); assertTrue(fileSystem.exists("/")); @@ -44,7 +44,7 @@ @Test public void testMultipleNodes2() throws Exception { - DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -61,7 +61,7 @@ assertArrayEquals(bytes, getResult); } - + /* @Test public void testMultipleNodes3() throws Exception { Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,346 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ConcurrentModificationException; -import java.util.List; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.util.FileUtil; - - -public abstract class DisworkFileSystemTest { - - /** - * a place to store files for the test it's a subdirectory of the OS temp - * dir e.g. under linux /tmp/disworkfs/tests/ - */ - static protected String tempDirectoryPath = - System.getProperty("java.io.tmpdir", ".") // temp directory - + "/disworkfs/tests"; - - /** - * We will create a file at this path for test purpose - */ - static protected String randomFilePath = tempDirectoryPath + "/randomfile"; - - /** - * The file will have this fixed size - */ - static protected int randomFileSize = 9999; - - static protected DisworkFileSystem fileSystem; - - /** - * This setUp creates in a temp directory a file of size - * {@link randomFileSize} (fulfilling it with random bytes) - * This file can be found at {@link #randomFilePath} - * At the end of the method {@link #fileSystem} is ready to be used - * @throws Exception - */ - @Before - public void setUp() throws Exception { - // create a temp directory for our test - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - // creating random data for the file - Random random = new Random(); - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - - } - - @After - public void tearDown() throws Exception { - // cleaning - FileUtil.deleteRecursively(tempDirectoryPath); - } - - /** - * writing a file at root directory should not raise any exception - * @throws Exception - */ - @Test - public void testWrite() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - } - - /** - * First, write a file then test if exists return true. - * @throws Exception - */ - @Test - public void testExists() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); - } - - /** - * try to read a file that as never been created nor written - */ - @Test(expected = FileNotFoundException.class) - public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); - } - - /** - * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} - * by storing a "-1" byte, can be buggy due to the use of read() - * @throws IOException - */ - @Test - public void testSplit() throws IOException { - - byte[] bytes = new byte[1]; - bytes[0] = -0x1; - - InputStream source; - - source = new ByteArrayInputStream(bytes); - fileSystem.write("/", "my_file", source); - - source.close(); - - - source = new ByteArrayInputStream(bytes); - InputStream readResult = fileSystem.read("/my_file"); - - int read = 0; - byte[] b = new byte[1]; - - read = readResult.read(b); - - assertEquals(1, read); - assertArrayEquals(bytes, b); - - } - - /** - * writing a file at the root directory and reading it. - * finally, compare original source and read result - * byte-to-byte: contents should be equals - * @throws Exception - */ - @Test - public void testWriteRead() throws Exception { - - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/", "my_file", source); - - source.close(); - - InputStream readResult; - - // now, the checks. We read the original file and the result of - // a read() and then compare it byte-to-byte - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - - assertEquals(randomFileSize, source.available()); - assertEquals(randomFileSize, readResult.available()); - - byte[] sourceAsBytes = IOUtils.toByteArray(source); - byte[] readResultAsBytes = IOUtils.toByteArray(readResult); - - assertArrayEquals(sourceAsBytes, readResultAsBytes); - - source.close(); - readResult.close(); - - } - - /** - * this use case should raise an exception because my_folder - * doesn't exists - */ - @Test(expected = IOException.class) - public void testFailAtWrite() throws Exception { - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - } - - /** - * this use case should raise an exception because writing to a file - * that already exists - */ - @Test(expected = IOException.class) - public void testFailAtDoubleWrite() throws Exception { - InputStream source = new FileInputStream(randomFilePath); - fileSystem.write("/my_file", source); - source.close(); - - source = new FileInputStream(randomFilePath); - try { - fileSystem.write("/my_file", source); - } finally { - source.close(); - } - } - - /** - * This test uses mkdir to create dirs and sub-dirs - * @throws Exception - */ - @Test - public void testCreateDirectory() throws Exception { - fileSystem.createDirectory("/my_folder"); - assertTrue(fileSystem.exists("/my_folder")); - fileSystem.createDirectory("/my_folder/my_sub_folder"); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); - } - - /** - * Create some folders with mkdir and write files in those directories - * @throws Exception - */ - @Test - public void testWriteInFolder() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.createDirectory("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder", "my_file", - new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); - } - - /** - * create a symbolic link to a file. This test show that we can read the - * file using the link - * @throws Exception - */ - @Test - public void testLinking() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); - - InputStream source = new FileInputStream(randomFilePath); - InputStream readResult = fileSystem.read("/my_link"); - - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); - } - - /** - * Trying to create a link to a wrong target, this sould raise an exception - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFailAtLinking() throws Exception { - fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); - } - - /** - * Trying to remove files and directories - * @throws Exception - */ - @Test - public void testRemove() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.delete("/my_folder/my_file"); - assertTrue(fileSystem.exists("/my_folder")); - assertFalse(fileSystem.exists("/my_folder/my_file")); - fileSystem.delete("/my_folder"); - assertFalse(fileSystem.exists("/my_folder")); - } - - /** - * By trying to remove a non-empty directory, this test should raise an - * exception - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFailAtRemove() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - - // trying to remove a non-empty directory should raise an exception - fileSystem.delete("/my_folder"); - } - - /** - * This tests uses ls - */ - @Test - public void testListDirectory() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.createDirectory("/my_folder/my_sub_dir"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); - - List<String> lsResult = fileSystem.readDirectory("/my_folder"); - - // checking that result contains all the required data - assertTrue(lsResult.contains("my_sub_dir")); - assertTrue(lsResult.contains("my_file")); - assertTrue(lsResult.contains("my_link")); - - // ... and only those - assertEquals(3, lsResult.size()); - - lsResult = fileSystem.readDirectory("/"); - assertEquals(1, lsResult.size()); - } - - @Test - public void testConcurrency() throws Exception { - fileSystem.createDirectory("/mydir"); - try { - fileSystem.createDirectory("/myseconddir"); - } catch (ConcurrentModificationException e) { - fail(); - } - } - - @Test - public void testLinks() throws Exception { - fileSystem.createDirectory("/dir"); - fileSystem.createDirectory("/dir/subdir"); - fileSystem.createDirectory("/otherdir"); - - fileSystem.createSymbolicLink("/link", "dir/subdir"); - fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); - List<String> readResult = fileSystem.readDirectory("/dir/subdir"); - assertEquals(1, readResult.size()); - assertTrue(readResult.contains("subsubdirlink")); - - fileSystem.createDirectory("/link/subsubdirlink/finaldir"); - readResult = fileSystem.readDirectory("/otherdir"); - assertEquals(1, readResult.size()); - assertTrue(readResult.contains("finaldir")); - - } -} Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,89 +0,0 @@ -package org.nuiton.disworkfs; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.Socket; -import java.net.UnknownHostException; - -/** - * This class provides utilities to write tests easily. - * - * You can get multiples diswork configs ready to use. All the instances uses - * a new port and the local IP. - * <pre> - * c = newDisworkConfig(); // create a config for a bootstrap node - * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that - * // will bootstrap by joining the - * // first node - * c3 = newDisworkConfig(c.getUsedPort()) - * </pre> - * - */ -public class Util { - - protected static Integer port = 19000; - - /** - * returns a new port, returned value change at each call. - * @return - */ - public static Integer getPort() { - port += 1; - return port; - } - - /** - * returns the IP on the local machine. Trying to get an public IP or a LAN - * IP or the loopback IP if there is no other interface - * @return - * @throws UnknownHostException - */ - public static InetAddress getIp() throws UnknownHostException { - InetAddress result = InetAddress.getLocalHost(); - if (result.isLoopbackAddress()) { - try { - Socket temp = new Socket("microsoft.com", 80); - result = temp.getLocalAddress(); - temp.close(); - } catch (IOException e) { - // TODO 20100602 bleny do something ? - } - } - return result; - } - - /** - * returns a @link {@link DisworkFileSystemConfig} ready to be use as a - * config for a single-node instance of DisworkFS - * @return - * @throws UnknownHostException - */ - public static DisworkFileSystemConfig newDisworkConfig() - throws UnknownHostException { - return newDisworkConfig(null); - } - - /** - * returns a @link {@link DisworkFileSystemConfig} ready to be use as a - * config for a multiple-node instance of DisworkFS on a same machine. - * @param bootstrapPort the port on the same machine where another node can - * be found to bootstrap - * @return a complete config - * @throws UnknownHostException - */ - public static DisworkFileSystemConfig newDisworkConfig(Integer bootstrapPort) - throws UnknownHostException { - DisworkFileSystemConfig result = new DisworkFileSystemConfig(); - String port = Util.getPort().toString(); - String ip = getIp().getHostAddress(); - result.setOption("diswork.fs.use_in_memory_map", "false"); - result.setOption("diswork.fs.use_port", port); - result.setOption("diswork.fs.bootstrap.ip", ip); - if (bootstrapPort == null) { - result.setOption("diswork.fs.bootstrap.port", port); - } else { - result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); - } - return result; - } -} \ No newline at end of file Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,106 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +public abstract class AbstractDisworkMapTest { + + private final Log log = LogFactory.getLog(AbstractDisworkMapTest.class); + + + protected DisworkMap map1; + protected DisworkMap map2; + + @Before + public void setUp() throws Exception { + + } + + @After + public void tearDown() throws Exception { + /* + if (map1 != null) + map1.close(); + if (map2 != null) + map2.close(); + */ + } + + /** + * this test show that the InMemoryMap put implies a copy of the data. + * After the put, the original value is modified. A final check shows + * there is no side effect + */ + @Test + public void testPut() { + byte[] expected = {0x1, 0x2}; + map1.put("key", expected); + expected[0] = 0xf; + byte[] actual = map1.get("key"); + assertFalse(Arrays.equals(expected, actual)); + } + + + @Test + public void test() throws Exception { + DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); + + map1 = new KademliaDisworkMap(config); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + map1.put("test", newBytes); + + byte[] getResult = map1.get("test"); + assertArrayEquals(newBytes, getResult); + } + + @Test + public void test2nodes() throws Exception { + DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); + map1 = new KademliaDisworkMap(config1); + + DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); + map2 = new KademliaDisworkMap(config2); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + log.info("trying put test"); + map1.put("test", newBytes); + + log.info("trying contains test"); + assertTrue(map2.containsKey("test")); + log.info("trying contains unknownkey"); + assertFalse(map2.containsKey("unknownkey")); + + log.info("trying get test"); + byte[] getResult = map2.get("test"); + + assertArrayEquals(newBytes, getResult); + + log.info("trying remove test"); + byte[] removeResult = map2.remove("test"); + assertArrayEquals(newBytes, removeResult); + + log.info("trying put test"); + map1.put("test", new byte[0]); + + log.info("trying put test"); + map2.put("test", new byte[1]); + + log.info("trying remove unknow"); + map2.remove("unknow"); + + } + +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java (from rev 60, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,14 @@ +package org.nuiton.disworkfs.storage; + +import org.junit.Before; + + +public class InMemoryDisworkMapTest extends AbstractDisworkMapTest { + + @Before + public void setUp() throws Exception { + map1 = new InMemoryDisworkMap(); + map2 = map1; + } + +} Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,27 +0,0 @@ -package org.nuiton.disworkfs.storage; - -import static org.junit.Assert.assertFalse; - -import java.util.Arrays; - -import org.junit.Test; -import org.nuiton.disworkfs.storage.InMemoryDisworkMap; - -public class InMemoryMapTest { - - /** - * this test show that the InMemoryMap put implies a copy of the data. - * After the put, the original value is modified. A final check shows - * there is no side effect - */ - @Test - public void testPut() { - InMemoryDisworkMap map = new InMemoryDisworkMap(); - byte[] expected = {0x1, 0x2}; - map.put("key", expected); - expected[0] = 0xf; - byte[] actual = map.get("key"); - assertFalse(Arrays.equals(expected, actual)); - } - -} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,31 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +public class KademliaDisworkMapTest extends AbstractDisworkMapTest { + + @Before + public void setUp() throws Exception { + DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); + map1 = new KademliaDisworkMap(config1); + + DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); + map2 = new KademliaDisworkMap(config2); + } + + @Test + public void testStringToIndentifier() { + assertNotNull(KademliaDisworkMap.stringToIdentifier("/")); + assertNotNull(KademliaDisworkMap.stringToIdentifier( + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" + /* + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" + + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir"*/)); + } +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,98 +1,17 @@ package org.nuiton.disworkfs.storage; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import org.junit.After; import org.junit.Before; -import org.junit.Test; import org.nuiton.disworkfs.DisworkFileSystemConfig; -import org.nuiton.disworkfs.Util; -public class PastryDisworkMapTest { - - protected static PastryDisworkMap map1; - protected static PastryDisworkMap map2; +public class PastryDisworkMapTest extends AbstractDisworkMapTest { - protected static byte[] bytes; - @Before public void setUp() throws Exception { - } - - @After - public void tearDown() throws Exception { - /* - if (map1 != null) - map1.close(); - if (map2 != null) - map2.close(); - */ - } - - @Test - public void test() throws Exception { - DisworkFileSystemConfig config = Util.newDisworkConfig(); - - map1 = new PastryDisworkMap(config); - - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - bytes = newBytes; - - map1.put("test", newBytes); - - byte[] getResult = map1.get("test"); - assertArrayEquals(newBytes, getResult); - } - - @Test - public void test2nodes() throws Exception { - DisworkFileSystemConfig config1 = Util.newDisworkConfig(); + DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newPastryDisworkConfig(); map1 = new PastryDisworkMap(config1); - DisworkFileSystemConfig config2 = Util.newDisworkConfig(config1.getUsedPort()); + DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newPastryDisworkConfig(config1.getUsedPort()); map2 = new PastryDisworkMap(config2); - - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - bytes = newBytes; - - map1.put("test", newBytes); - - assertTrue(map2.containsKey("test")); - assertFalse(map2.containsKey("unknownkey")); - - byte[] getResult = map2.get("test"); - - assertArrayEquals(newBytes, getResult); - - byte[] removeResult = map2.remove("test"); - assertArrayEquals(newBytes, removeResult); - - map1.put("test", newBytes); - map2.put("test", newBytes); - - map2.remove("unknow"); - } - @Test - public void testClose() throws Exception { - /* - DisworkFileSystemConfig config1 = new DisworkFileSystemConfig(); - map1 = new PastryDisworkMap(config1); - */ - /* - try { - map2 = new PastryDisworkMap(config1); - } catch (IllegalStateException e) { - assertEquals("Cannot bind to /192.168.99.119:9001", e.getMessage()); - } - */ - - // map1.close(); - // map2 = new PastryDisworkMap(config1); - } - } Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/pom.xml 2010-06-04 18:15:37 UTC (rev 62) @@ -78,6 +78,11 @@ <artifactId>pastry</artifactId> <version>2.1</version> </dependency> + <dependency> + <groupId>org.planx.xmlstore</groupId> + <artifactId>xmlstore</artifactId> + <version>0.4.12</version> + </dependency>  <dependency>

1 0

r61 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 18:25:47 +0200 (Thu, 03 Jun 2010) New Revision: 61 Url: http://nuiton.org/repositories/revision/diswork/61 Log: bug fixes Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 16:25:47 UTC (rev 61) @@ -88,15 +88,14 @@ return finished; } - public void waitUntilFinised() { + public void waitUntilFinished() { while(!finished) { try { Thread.sleep(1500); log.info("waiting for response"); } catch (InterruptedException e) { - // TODO 20100527 bleny stub - log.info("exception catch", e); - e.printStackTrace(); + throw new RuntimeException + ("interrupted while waiting for response", e); } } } @@ -147,7 +146,12 @@ try { factory = new SocketPastryNodeFactory(nidFactory, bindport, env); } catch (BindException e) { - throw new IOException("unable to bind to port" + bindport); + // the bootstrap node can't be joined + throw new IOException(""/*"bootstrap node can't be joined"*/, e); + } catch (IllegalStateException e) { + // the bootstrap node can't be joined + throw new IOException + ("unable to bind to already used port " + bindport, e); } catch (ConnectException e) { // this occurs some times, it may work after some time... numberOfTry += 1; @@ -229,7 +233,7 @@ Id id = pastryIdFactory.buildId((String) key); ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation(); past.existsInOverlay(id, containsKeyContinuation); - containsKeyContinuation.waitUntilFinised(); + containsKeyContinuation.waitUntilFinished(); boolean result = containsKeyContinuation.result; log.info("containsKey " + key + " (id = " + id + ") returns " + result); return result; @@ -247,13 +251,19 @@ } finished = true; } + + @Override + public void receiveException(Exception exception) { + log.error("exception received while getting : " + exception); + finished = true; + } } protected byte[] atomicGet(Object key) { Id id = pastryIdFactory.buildId((String) key); GetContinuation getContinuation = new GetContinuation(); past.lookup(id, getContinuation); - getContinuation.waitUntilFinised(); + getContinuation.waitUntilFinished(); return getContinuation.result; } @@ -274,6 +284,12 @@ log.info("insert result received : " + Arrays.toString(result)); finished = true; } + + @Override + public void receiveException(Exception exception) { + log.error("exception received while inserting : " + exception); + finished = true; + } } protected void atomicPut(String key, byte[] value) { @@ -281,7 +297,7 @@ PastContent pastContent = new ByteArrayPastContent(id, value); PutContinuation putContinuation = new PutContinuation(); past.insert(pastContent, putContinuation); - putContinuation.waitUntilFinised(); + putContinuation.waitUntilFinished(); } @Override @@ -305,13 +321,19 @@ this.result = result; finished = true; } + + @Override + public void receiveException(Exception exception) { + log.error("exception received while removing : " + exception); + finished = true; + } } protected void atomicRemove(Object key) { Id id = pastryIdFactory.buildId((String) key); RemoveContinuation removeContinuation = new RemoveContinuation(); past.remove(id, removeContinuation); - removeContinuation.waitUntilFinised(); + removeContinuation.waitUntilFinished(); log.info("atomic remove " + key + " has returned " + removeContinuation.result); } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 15:09:54 UTC (rev 60) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 16:25:47 UTC (rev 61) @@ -308,6 +308,8 @@ map.put(key, map.get(newDataKey)); } + map.remove(newDataKey); + unLock(key); } else { Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-03 15:09:54 UTC (rev 60) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-03 16:25:47 UTC (rev 61) @@ -21,7 +21,7 @@ */ public class Util { - protected static Integer port = 9000; + protected static Integer port = 19000; /** * returns a new port, returned value change at each call.

1 0

r60 - in trunk: diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit diswork-fs/src/main/java/org/nuiton/disworkfs/storage src/license
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 17:09:54 +0200 (Thu, 03 Jun 2010) New Revision: 60 Url: http://nuiton.org/repositories/revision/diswork/60 Log: license headers, licence descriptor Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java trunk/src/license/project.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs; import java.io.IOException; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs; import java.io.Closeable; @@ -26,14 +50,12 @@ * </ul> */ public class DisworkFileSystem implements Closeable { + + private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - /** - * storage will permit to save and read directories, files and links - */ + /** storage will permit to save and read directories, files and links */ protected Storage storage; - private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - /** number of try to acquire a lock before giving up */ protected static final int LOCK_MAX_NUMBER_OF_TRY = 10; @@ -41,7 +63,7 @@ protected static final int LOCK_WAIT = 10 * 1000; // ten seconds /** default constructor (uses default configuration parameters) - * @throws IOException + * @throws IOException caused by network issue */ public DisworkFileSystem() throws IOException { this(new DisworkFileSystemConfig()); @@ -50,7 +72,7 @@ /** constructor allowing to provide another configuration to use * * @param config the configuration - * @throws IOException + * @throws IOException caused by network issue */ public DisworkFileSystem(DisworkFileSystemConfig config) throws IOException { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs; import org.nuiton.util.ApplicationConfig; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ /** * DisworkFS is a distributed file system. You can use it by instantiating * {@link org.nuiton.disworkfs.DisworkFileSystem} and use it to store Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.peerunit; import static fr.inria.peerunit.test.assertion.Assert.assertTrue; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.Closeable; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.IOException; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.IOException; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.ByteArrayInputStream; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ /** * * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a Modified: trunk/src/license/project.xml =================================================================== --- trunk/src/license/project.xml 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/src/license/project.xml 2010-06-03 15:09:54 UTC (rev 60) @@ -9,7 +9,7 @@ <commentStyle>java</commentStyle> <fileSets> <fileSet> - <basedir>all/src</basedir> + <basedir>src/main/java</basedir> <includes> <include>**/*.java</include> </includes>

1 0

r59 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 16:35:49 +0200 (Thu, 03 Jun 2010) New Revision: 59 Url: http://nuiton.org/repositories/revision/diswork/59 Log: concurrency, - memory leak Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 13:11:32 UTC (rev 58) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 14:35:49 UTC (rev 59) @@ -137,9 +137,9 @@ if (EntryUtil.isDirectory(entryParent)) { String parentId = EntryUtil.getIdFromEntry(entryParent); + + // checking that file do not already exists in this directory String content = storage.getDirectory(parentId); - - // checking that file do not already exists in this directory String findResult = EntryUtil.findEntryInDirectory (content, fileName); if (findResult != null) { @@ -147,11 +147,8 @@ (parent + " already contains an element named " + fileName); } + // file do not exists, write file on the FS String newFileId = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.F, fileName, newFileId); - - // store file before meta info storage.putFile(newFileId, source); // update directory content @@ -160,6 +157,10 @@ while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { lockAcquired = storage.tryToLock(parentId); if (lockAcquired) { + // parent dir is locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); storage.putDirectory(parentId, newContent); storage.unLock(parentId); } else { @@ -167,7 +168,7 @@ try { Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { - log.info("exception catch", e); + log.info("wait for lock interrupted", e); throw new IOException ("interrupted while trying to acquire lock", e); } @@ -219,16 +220,49 @@ if (EntryUtil.isDirectory(entryParent)) { String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, dirName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + dirName); + } + + // store file before meta info String newDirectoryId = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.D, dirName, newDirectoryId); - - // store file before meta info storage.putDirectory(newDirectoryId, EntryUtil.EMPTY_DIRECTORY_CONTENT); - // update meta info directory - storage.putDirectory(parentId, newContent); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } } else if (EntryUtil.isLink(entryParent)) { String linkTarget = storage.getLink( EntryUtil.getIdFromEntry(entryParent)); @@ -282,19 +316,55 @@ } if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, name); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + name); + } + + // store file before meta info String newLinkId = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.L, name, newLinkId); + storage.putLink(newLinkId, target); - // store link before meta info - storage.putLink(newLinkId, target); - // update meta info directory - storage.putDirectory(parentId, newContent); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.L, name, newLinkId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { String linkTarget = storage.getLink( - EntryUtil.getIdFromEntry(entryParent)); + EntryUtil.getIdFromEntry(entryParent)); String newTarget = EntryUtil.resolveLink(parent, linkTarget); createSymbolicLink(newTarget, name, target); } else if (EntryUtil.isFile(entryParent)) { @@ -328,9 +398,6 @@ * @see #delete(String) */ protected void delete(String parent, String name) throws IOException { - // FIXME 20100602 bleny this implementation leaks - // only the association path => id of the metablock is removed, all - // block remains String entryParent = walk(parent); if (entryParent == null) { @@ -344,10 +411,9 @@ String entry = EntryUtil.findEntryInDirectory(content, name); String idToRemove = EntryUtil.getIdFromEntry(entry); - String newContent = EntryUtil.removeEntryFromEntries(content, name); - - // check if not removing a non-empty directory + // according to the type of the entry name, delete it if (EntryUtil.isDirectory(entry)) { + // check if not removing a non-empty directory String innerDirectoryId = EntryUtil.getIdFromEntry(entry); String innerDirectoryContent = storage.getDirectory(innerDirectoryId); @@ -358,13 +424,49 @@ throw new IOException ("trying to remove a non-empty directory"); } + + // remove it + storage.removeDirectory(idToRemove); + + } else if (EntryUtil.isFile(entry)) { + storage.removeFile(idToRemove); + } else if (EntryUtil.isLink(entry)) { + storage.removeLink(idToRemove); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } - // update meta info directory - storage.putDirectory(parentId, newContent); - // store file before meta info - storage.remove(idToRemove); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.removeEntryFromEntries(content, name); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { String linkTarget = storage.getLink( EntryUtil.getIdFromEntry(entryParent)); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 13:11:32 UTC (rev 58) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 14:35:49 UTC (rev 59) @@ -9,8 +9,6 @@ import java.util.List; import java.util.Map; -import javax.swing.RowFilter.Entry; - import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -196,7 +194,22 @@ map.put(id, contentAsBytes); } + public void removeDirectory(String id) throws IOException { + log.debug("removeDirectory(\"" + id + "\")"); + removeKey(id); + } + public void removeFile(String id) throws IOException { + log.debug("removeFile(\"" + id + "\")"); + removeKey(id); + } + + public void removeLink(String id) { + log.debug("removeLink(\"" + id + "\")"); + map.remove(id); + } + + /** * see {@link #get(String)} */ @@ -213,12 +226,9 @@ * @param value * @throws IOException */ - protected void put(String key, InputStream value) + protected void put(String key, InputStream value) throws IOException, ConcurrentModificationException { - // TODO 20100602 bleny locks should be signed to prevent concurrency : - // due to latency, a lock claimed may be obtained even if someone tryed - // to obtain it before boolean lockAcquired = tryToLock(key); if (lockAcquired) { @@ -269,8 +279,11 @@ } eraseDependenciesOfMetablock(key); - map.put(key, map.get(newDataKey)); + if (isLockStillOwned(key)) { + map.put(key, map.get(newDataKey)); + } + unLock(key); } else { @@ -279,12 +292,18 @@ } + protected void remove(String key) throws IOException, + ConcurrentModificationException { + eraseDependenciesOfMetablock(key); + removeKey(key); + } + /** * due to concurrency, key should not be removed immediately. Someone may * read it now. It will be removed later, using {@link #clean()}. * @param key the key to be removed from the map */ - public void remove(String key) { + public void removeKey(String key) { idsToBeRemoved.add(key); } @@ -314,7 +333,7 @@ log.info("trying to acquire a lock on " + key); String lockKey = keyToLockKey(key); - byte[] lock = map.put(lockKey, EntryUtil.newLock(ownerId)); + byte[] lock = map.put(lockKey, EntryUtil.newLock(ownerId)); Boolean result = null; // if there was no lock or if current lock is mine @@ -356,7 +375,7 @@ * the value at this key has to be a metablock * @param key */ - public void eraseDependenciesOfMetablock(String key) { + protected void eraseDependenciesOfMetablock(String key) { byte[] value = map.get(key); if (value != null) { String obsoleteMetaBlock = EntryUtil.bytesToString(value); @@ -365,11 +384,19 @@ EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); log.info("removing " + obsoleteBlocksIds.length + " old blocks"); for (String obsoleteBlockId : obsoleteBlocksIds) { - map.remove(obsoleteBlockId); + removeKey(obsoleteBlockId); } } } } + + private boolean isLockStillOwned(String key) { + String lockKey = keyToLockKey(key); + byte[] lock = map.get(lockKey); + String lockOwner = EntryUtil.getOwnerFromLock(lock); + boolean isLockOwned = ownerId.equals(lockOwner); + return isLockOwned; + } protected boolean isObsolete(byte[] lock) { Long currentDate = System.currentTimeMillis(); Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 13:11:32 UTC (rev 58) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 14:35:49 UTC (rev 59) @@ -341,6 +341,6 @@ readResult = fileSystem.readDirectory("/otherdir"); assertEquals(1, readResult.size()); assertTrue(readResult.contains("finaldir")); - + } }

1 0

r58 - in trunk: diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/storage diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage src src/license
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 15:11:32 +0200 (Thu, 03 Jun 2010) New Revision: 58 Url: http://nuiton.org/repositories/revision/diswork/58 Log: licence plugin config ; some bugfixs Added: trunk/src/license/ trunk/src/license/project.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 13:11:32 UTC (rev 58) @@ -34,6 +34,12 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); + /** number of try to acquire a lock before giving up */ + protected static final int LOCK_MAX_NUMBER_OF_TRY = 10; + + /** time (ms) to wait between two try to acquire a lock */ + protected static final int LOCK_WAIT = 10 * 1000; // ten seconds + /** default constructor (uses default configuration parameters) * @throws IOException */ @@ -147,10 +153,42 @@ // store file before meta info storage.putFile(newFileId, source); - // update meta info directory - storage.putDirectory(parentId, newContent); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("exception catch", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + write(newTarget, fileName, source); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } } @@ -187,12 +225,20 @@ // store file before meta info storage.putDirectory(newDirectoryId, - EntryUtil.newEmptyDirectoryContent()); + EntryUtil.EMPTY_DIRECTORY_CONTENT); // update meta info directory storage.putDirectory(parentId, newContent); + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createDirectory(newTarget, dirName); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } } @@ -217,13 +263,16 @@ protected void createSymbolicLink(String parent, String name, String target) throws IOException { - if (target.startsWith("/")) { + if (target.startsWith(EntryUtil.PATH_SEPARATOR)) { // target is absolute } else { // target is relative, taking this in consideration target = EntryUtil.resolveLink(parent, target); } + log.info("trying to create symbolic link named " + name + " at path " + + parent + " with target \"" + target + "\""); + if (exists(target)) { String entryParent = walk(parent); @@ -243,8 +292,16 @@ storage.putLink(newLinkId, target); // update meta info directory storage.putDirectory(parentId, newContent); + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createSymbolicLink(newTarget, name, target); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } @@ -271,6 +328,9 @@ * @see #delete(String) */ protected void delete(String parent, String name) throws IOException { + // FIXME 20100602 bleny this implementation leaks + // only the association path => id of the metablock is removed, all + // block remains String entryParent = walk(parent); if (entryParent == null) { @@ -284,8 +344,7 @@ String entry = EntryUtil.findEntryInDirectory(content, name); String idToRemove = EntryUtil.getIdFromEntry(entry); - String newContent = EntryUtil.removeEntryFromEntries( - content, name); + String newContent = EntryUtil.removeEntryFromEntries(content, name); // check if not removing a non-empty directory if (EntryUtil.isDirectory(entry)) { @@ -294,7 +353,7 @@ storage.getDirectory(innerDirectoryId); // checking the emptiness of the directory if (!innerDirectoryContent.equals( - EntryUtil.newEmptyDirectoryContent())) { + EntryUtil.EMPTY_DIRECTORY_CONTENT)) { // directory is not empty throw new IOException ("trying to remove a non-empty directory"); @@ -306,8 +365,16 @@ // store file before meta info storage.remove(idToRemove); + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + delete(newTarget, name); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } } @@ -335,7 +402,7 @@ result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); - if (EntryUtil.newEmptyDirectoryContent().equals(content)) { + if (EntryUtil.EMPTY_DIRECTORY_CONTENT.equals(content)) { // directory is empty, add nothing } else { String[] entries = content.split( @@ -380,10 +447,10 @@ // FIXME 20105021 bleny works fine but is not understandable String result = null; - if (path.equals("/")) { + if (path.equals(EntryUtil.ROOT_DIRECTORY)) { return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "/" + EntryUtil.ENTRY_SEPARATOR - + "/"; + + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY; } String parentPath = EntryUtil.getParentFromPath(path); @@ -391,7 +458,7 @@ if (content == null) { // start of walk content = storage.getRootDirectory(); - result = walk(path, "/", content); + result = walk(path, EntryUtil.ROOT_DIRECTORY, content); } else if (parentPath.equals(current)) { // in the last directory // recuperation de l'element a traiter dans p @@ -414,7 +481,7 @@ String tail; // the path still remaining when in current - if (current.equals("/")) { + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { tail = path.substring(current.length()); } else { tail = path.substring(current.length() + 1); @@ -422,14 +489,16 @@ log.debug("current = " + current); log.debug("tail = " + tail); - String[] elementsNames = tail.split("/"); + String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); String p = elementsNames[0]; log.info("in intermediate dir " + current + ", looking for " + p); // mise a jour de current - if (current.equals("/")) current = ""; // avoid "//path" next line - current += "/" + p; + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + current = ""; // avoid "//path" next line + } + current += EntryUtil.PATH_SEPARATOR + p; String entry = EntryUtil.findEntryInDirectory(content, p); if (entry == null) { @@ -463,6 +532,6 @@ @Override public void close() throws IOException { storage.close(); - } + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 13:11:32 UTC (rev 58) @@ -32,21 +32,25 @@ public class EntryUtil { static public enum TYPE { /** Directory */ D, - /** Link */ L, - /** File */ F + /** Link */ L, + /** File */ F }; - static final int TYPE_LENGTH = 1; + static final public int TYPE_LENGTH = 1; static final public String ENTRY_SEPARATOR = ":"; static final public String ENTRIES_SEPARATOR = "\n"; static final public String BLOCKIDS_SEPARATOR = ";"; + static final public String PATH_SEPARATOR = "/"; + static final public String ROOT_DIRECTORY = "/"; + static final public String EMPTY_DIRECTORY_CONTENT = ""; + static final public String LOCK_SEPARATOR = "%"; - static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); + static final public int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); /** * This charset is forced in string-to-bytes and bytes-to-string conversions */ - static protected Charset CHARSET = Charset.forName("UTF-8"); + static final protected Charset CHARSET = Charset.forName("UTF-8"); /** * This is a utility class thus should not be instantiated @@ -141,13 +145,6 @@ /** * - */ - static public String newEmptyDirectoryContent() { - return ""; - } - - /** - * * @param content the string content of the directory * (all entries before add) * @param type the type of the new entry @@ -164,10 +161,15 @@ static public int getTotalSizeFromMetaBlock(String metaBlock) { // dealing with empty meta-block (for empty directory or file) - if (metaBlock.equals("0")) return 0; - String result = metaBlock.substring(0, - metaBlock.indexOf(BLOCKIDS_SEPARATOR)); - return Integer.valueOf(result); + Integer result = null; + if (metaBlock.equals("0")) { + result = 0; + } else { + String resultAsString = metaBlock.substring(0, + metaBlock.indexOf(BLOCKIDS_SEPARATOR)); + result = Integer.valueOf(resultAsString); + } + return result; } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { @@ -218,7 +220,7 @@ public static String getParentFromPath(String path) { String parent = FilenameUtils.getFullPathNoEndSeparator(path); if (parent.isEmpty()) { - parent = "/"; + parent = ROOT_DIRECTORY; } return parent; } @@ -227,4 +229,25 @@ return FilenameUtils.getName(path); } + public static String getOwnerFromLock(byte[] lock) { + String lockAsString = bytesToString(lock); + String lockInfo[] = lockAsString.split(LOCK_SEPARATOR); + String owner = lockInfo[0]; + return owner; + } + + public static Long getTimeFromLock(byte[] lock) { + String lockAsString = bytesToString(lock); + String lockInfo[] = lockAsString.split(LOCK_SEPARATOR); + Long lockAge = Long.parseLong(lockInfo[1]); + return lockAge; + } + + public static byte[] newLock(String ownerId) { + Long currentDate = System.currentTimeMillis(); + String lock = ownerId + LOCK_SEPARATOR + currentDate; + byte[] result = EntryUtil.stringToBytes(lock); + return result; + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 13:11:32 UTC (rev 58) @@ -1,6 +1,8 @@ package org.nuiton.disworkfs.storage; import java.io.IOException; +import java.net.BindException; +import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.Arrays; @@ -120,15 +122,17 @@ while(factory == null && numberOfTry <= 10) { try { factory = new SocketPastryNodeFactory(nidFactory, bindport, env); - } catch (java.net.ConnectException e) { + } catch (BindException e) { + throw new IOException("unable to bind to port" + bindport); + } catch (ConnectException e) { + // this occurs some times, it may work after some time... numberOfTry += 1; + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException ee) { + throw new IOException("Pastry boot process interrupted", ee); + } } - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - log.info("exception catch", e); - e.printStackTrace(); - } } if (factory == null) { throw new IOException("unable to connect"); @@ -147,16 +151,14 @@ try { node.wait(500); } catch (InterruptedException e) { - // TODO 20100528 bleny Auto-generated catch block - log.info("exception catch", e); - e.printStackTrace(); + throw new IOException("Pastry boot process interrupted", e); } // abort if can't join if (node.joinFailed()) { throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason()); } - } + } } log.info("finished creating new node " + node); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 13:11:32 UTC (rev 58) @@ -9,6 +9,8 @@ import java.util.List; import java.util.Map; +import javax.swing.RowFilter.Entry; + import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -27,6 +29,18 @@ protected static final long LOCK_VALID_TIME = 60 * 60 * 1000; + protected String ownerId = EntryUtil.generateId(); + + protected static String keyToLockKey(String key) { + return key + "_lock"; + } + + protected static String keyToNewDataKey(String key) { + return key + "_new_data"; + } + + // FIXME 20100602 bleny this collection should be stored somewhere in + // the map so another node will perform remove if this one fail protected List<String> idsToBeRemoved = new ArrayList<String>(); /** @@ -44,8 +58,7 @@ protected ByteArrayInputStream currentBlock; - protected int available; - + protected int numberOfAvailableBytes; public SplitBlocksInputStream(Map<String, byte[]> map, String id) { super(); @@ -75,11 +88,11 @@ } int result = currentBlock.read(); - available -= 1; + numberOfAvailableBytes -= 1; if(log.isDebugEnabled()) { log.debug("reading block number " + blockIdsIndex + - " (available = " + available + ")" + + " (available = " + numberOfAvailableBytes + ")" + " returns " + result); } return result; @@ -93,9 +106,9 @@ String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; - available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + numberOfAvailableBytes = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); log.debug("initializing stream with meta block \"" + metaBlock - + "\" (" + blockIds.length + " blocks, " + available + + "\" (" + blockIds.length + " blocks, " + numberOfAvailableBytes + " bytes)"); } } @@ -103,7 +116,7 @@ @Override public int available() { checkInitialization(); - return available; + return numberOfAvailableBytes; } } @@ -114,17 +127,21 @@ public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { this.disworkConfig = disworkConfig; + + // instantiating a map according to config directives if (disworkConfig.useInMemoryMap()) { log.info("using in-memory map"); - this.map = new InMemoryDisworkMap(); - putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + map = new InMemoryDisworkMap(); } else { log.info("using Pastry map"); - this.map = new PastryDisworkMap(disworkConfig); - if (! map.containsKey("/")) { - putDirectory("/", EntryUtil.newEmptyDirectoryContent()); - } + map = new PastryDisworkMap(disworkConfig); } + + // creating root directory, if needed + if (! map.containsKey(EntryUtil.ROOT_DIRECTORY)) { + putDirectory(EntryUtil.ROOT_DIRECTORY, + EntryUtil.EMPTY_DIRECTORY_CONTENT); + } } /** @@ -132,7 +149,7 @@ * @throws IOException */ public String getRootDirectory() throws IOException { - String result = getDirectory("/"); + String result = getDirectory(EntryUtil.ROOT_DIRECTORY); return result; } @@ -189,7 +206,9 @@ } /** - * a put in the map, this involves split considerations and concurrency + * a put in the map, this involves split considerations and concurrency. + * a lock is acquired (it might have been acquired before). + * Lock is released at the end * @param key * @param value * @throws IOException @@ -197,90 +216,67 @@ protected void put(String key, InputStream value) throws IOException, ConcurrentModificationException { + // TODO 20100602 bleny locks should be signed to prevent concurrency : + // due to latency, a lock claimed may be obtained even if someone tryed + // to obtain it before + boolean lockAcquired = tryToLock(key); - String lockKey = key + "_lock"; - String newDataKey = key + "_newData"; - - // trying to acquire lock - Long currentDate = System.currentTimeMillis(); - log.info("trying to acquire a lock on " + key); - byte[] lock = map.put(lockKey, - EntryUtil.stringToBytes(currentDate.toString())); - - if (lock != null) { - // file is locked, check date - Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock)); - Long lockAge = currentDate - currentLock; - if (lockAge > LOCK_VALID_TIME) { - log.info("lock is out-dated (" + lockAge + " ms old)"); - // this lock is out-dated, let's erase all data - String obsoleteMetaBlock = - EntryUtil.bytesToString(map.get(newDataKey)); - if (obsoleteMetaBlock != null) { - String[] obsoleteBlocksIds = - EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); - log.info("removing " + obsoleteBlocksIds.length + " old blocks"); - for (String obsoleteBlockId : obsoleteBlocksIds) { - map.remove(obsoleteBlockId); - } - } - } else { - log.info(key + " is currently written, stopping operation"); - map.put(lockKey, lock); - throw new ConcurrentModificationException("can't write due to concurrency"); - } - } - - log.info("lock on " + key + " acquired"); - - // here, we know we can write or an exception - // would have been thrown earlier - String blocksIds = ""; - int readResult = 0; - int totalSize = 0; + if (lockAcquired) { + log.info("lock on " + key + " acquired"); - String metaBlock = totalSize + blocksIds; - map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); - - // creating a buffer of the size of a block - int bufferSize = disworkConfig.getBlockSize(); - byte[] buffer = new byte[bufferSize]; + // here, we know we can write or an exception + // would have been thrown earlier + String blocksIds = ""; + int readResult = 0; + int totalSize = 0; - while ((readResult = value.read(buffer)) != -1) { - totalSize += readResult; + String metaBlock = totalSize + blocksIds; + String newDataKey = keyToNewDataKey(key); + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // creating a buffer of the size of a block + int bufferSize = disworkConfig.getBlockSize(); + byte[] buffer = new byte[bufferSize]; - byte[] newBlock = buffer; + while ((readResult = value.read(buffer)) != -1) { + totalSize += readResult; - // if the buffer is not full, truncate the block - if (readResult < buffer.length) { - newBlock = new byte[readResult]; - System.arraycopy(buffer, 0, newBlock, 0, readResult); - } + byte[] newBlock = buffer; - String id = EntryUtil.generateId(); - blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + // if the buffer is not full, truncate the block + if (readResult < buffer.length) { + newBlock = new byte[readResult]; + System.arraycopy(buffer, 0, newBlock, 0, readResult); + } - log.debug("saving new block (size = " + newBlock.length + ")" + String id = EntryUtil.generateId(); + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + log.debug("saving new block (size = " + newBlock.length + ")" + " at key " + id); - // copy block in map - map.put(id, newBlock); + // copy block in map + map.put(id, newBlock); - metaBlock = totalSize + blocksIds; + metaBlock = totalSize + blocksIds; - log.debug("putting metablock " + metaBlock + " at key " + key); + log.debug("putting metablock " + metaBlock + " at key " + key); + + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // updating lock + updateLock(key); + } - map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + eraseDependenciesOfMetablock(key); + map.put(key, map.get(newDataKey)); - // updating lock - currentDate = System.currentTimeMillis(); - map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString())); + unLock(key); + + } else { + throw new ConcurrentModificationException("key " + key + " is locked"); } - map.put(key, map.get(newDataKey)); - - log.info("release lock " + lockKey); - map.remove(lockKey); } /** @@ -307,5 +303,79 @@ clean(); map.close(); } + + /** + * + * @param key + * @return true if lock acquired, false is lock is owned by another + */ + public boolean tryToLock(String key) { + // trying to acquire lock + log.info("trying to acquire a lock on " + key); + String lockKey = keyToLockKey(key); + byte[] lock = map.put(lockKey, EntryUtil.newLock(ownerId)); + + Boolean result = null; + // if there was no lock or if current lock is mine + if (lock == null || (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { + // file is not locked, we have took the lock + result = true; + } else { + // file is locked by other node, check date + if (isObsolete(lock)) { + log.info("lock is out-dated"); + // this lock is out-dated, let's erase all data + eraseNewData(key); + result = true; + } else { + log.info(key + " is currently written, stopping operation"); + map.put(lockKey, lock); + result = false; + } + } + return result; + } + + protected void updateLock(String key) { + String lockKey = keyToLockKey(key); + map.put(lockKey, EntryUtil.newLock(ownerId)); + } + + public void unLock(String key) { + String lockKey = keyToLockKey(key); + map.remove(lockKey); + } + + protected void eraseNewData(String key) { + String newDataKey = keyToNewDataKey(key); + eraseDependenciesOfMetablock(newDataKey); + } + + /** + * the value at this key has to be a metablock + * @param key + */ + public void eraseDependenciesOfMetablock(String key) { + byte[] value = map.get(key); + if (value != null) { + String obsoleteMetaBlock = EntryUtil.bytesToString(value); + if (obsoleteMetaBlock != null) { + String[] obsoleteBlocksIds = + EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); + log.info("removing " + obsoleteBlocksIds.length + " old blocks"); + for (String obsoleteBlockId : obsoleteBlocksIds) { + map.remove(obsoleteBlockId); + } + } + } + } + + protected boolean isObsolete(byte[] lock) { + Long currentDate = System.currentTimeMillis(); + Long currentLockTime = EntryUtil.getTimeFromLock(lock); + Long lockAge = currentDate - currentLockTime; + return lockAge > LOCK_VALID_TIME; + } + } \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-03 13:11:32 UTC (rev 58) @@ -17,15 +17,12 @@ public class DisworkFileSystemPastryTest extends DisworkFileSystemTest { + protected Integer bootstrapPort; + /** * this code executed after {@link DisworkFileSystemTest#setUp()} * @throws Exception */ - - private static final Log log = LogFactory.getLog(DisworkFileSystemPastryTest.class); - - protected Integer bootstrapPort; - @Before public void setUpFileSystem() throws Exception { // finally, initiate the fileSystem Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 13:11:32 UTC (rev 58) @@ -324,4 +324,23 @@ fail(); } } + + @Test + public void testLinks() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/otherdir"); + + fileSystem.createSymbolicLink("/link", "dir/subdir"); + fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); + List<String> readResult = fileSystem.readDirectory("/dir/subdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subsubdirlink")); + + fileSystem.createDirectory("/link/subsubdirlink/finaldir"); + readResult = fileSystem.readDirectory("/otherdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("finaldir")); + + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-03 13:11:32 UTC (rev 58) @@ -130,6 +130,15 @@ } @Test + public void testGetParentFromPath() { + String path; + path = "/file"; + assertEquals("/", EntryUtil.getParentFromPath(path)); + path = "/dir/file"; + assertEquals("/dir", EntryUtil.getParentFromPath(path)); + } + + @Test public void testGetTotalSizeFromMetaBlock() { assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); } Added: trunk/src/license/project.xml =================================================================== --- trunk/src/license/project.xml (rev 0) +++ trunk/src/license/project.xml 2010-06-03 13:11:32 UTC (rev 58) @@ -0,0 +1,20 @@ +<?xml version='1.0' encoding='UTF-8'?> +<project xmlns="http://maven-site.nuiton.org/maven-license-plugin/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven-site.nuiton.org/maven-license-plugin/1.0.0 http://maven-site.nuiton.org/maven-license-plugin/licenseProjectDescriptor-…"> + <mainLicense>lgpl_v3</mainLicense> + <headers> + <header> + <licenseName>lgpl_v3</licenseName> + <commentStyle>java</commentStyle> + <fileSets> + <fileSet> + <basedir>all/src</basedir> + <includes> + <include>**/*.java</include> + </includes> + </fileSet> + </fileSets> + </header> + </headers> +</project>

1 0

r57 - trunk/diswork-fs/src/main/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 12:23:17 +0200 (Thu, 03 Jun 2010) New Revision: 57 Url: http://nuiton.org/repositories/revision/diswork/57 Log: remove bad import Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-02 13:08:02 UTC (rev 56) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 10:23:17 UTC (rev 57) @@ -2,17 +2,12 @@ import java.io.IOException; import java.io.InputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.List; import java.util.Random; -import javax.swing.plaf.SliderUI; - import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.junit.runner.Runner; public class Demo { @@ -92,8 +87,7 @@ } catch (InterruptedException e) { log.info("sleep interrupted", e); } - - } + } } }

1 0

r56 - in trunk: . diswork-fs src/site src/site/rst/diswork-fs
by bleny＠users.nuiton.org June 2, 2010

June 2, 2010

Author: bleny Date: 2010-06-02 15:08:02 +0200 (Wed, 02 Jun 2010) New Revision: 56 Url: http://nuiton.org/repositories/revision/diswork/56 Log: cleaning pom, some documentation Added: trunk/src/site/rst/diswork-fs/how_it_works.rst Modified: trunk/diswork-fs/pom.xml trunk/pom.xml trunk/src/site/site_en.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-02 12:56:07 UTC (rev 55) +++ trunk/diswork-fs/pom.xml 2010-06-02 13:08:02 UTC (rev 56) @@ -1,62 +1,48 @@ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.nuiton</groupId> - <artifactId>diswork</artifactId> - <version>0.0.1-SNAPSHOT</version> - </parent> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.nuiton</groupId> + <artifactId>diswork</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> - <groupId>org.nuiton.diswork</groupId> - <artifactId>diswork-fs</artifactId> + <groupId>org.nuiton.diswork</groupId> + <artifactId>diswork-fs</artifactId> - <packaging>jar</packaging> - <name>disworkfs</name> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>2.0.2</version> - <configuration> - <source>1.6</source> - <target>1.6</target> - </configuration> - </plugin> - </plugins> - </build> - <dependencies> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>org.nuiton</groupId> - <artifactId>nuiton-utils</artifactId> - </dependency> - <dependency> - <groupId>org.freepastry</groupId> - <artifactId>pastry</artifactId> - </dependency> - <dependency> - <groupId>fr.inria.peerunit</groupId> - <artifactId>PeerUnit</artifactId> - </dependency> + <packaging>jar</packaging> + <name>disworkfs</name> + <dependencies> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + </dependency> + <dependency> + <groupId>org.freepastry</groupId> + <artifactId>pastry</artifactId> + </dependency> + <dependency> + <groupId>fr.inria.peerunit</groupId> + <artifactId>PeerUnit</artifactId> + </dependency> -  - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> -</project> - +  + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-02 12:56:07 UTC (rev 55) +++ trunk/pom.xml 2010-06-02 13:08:02 UTC (rev 56) @@ -1,11 +1,12 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> + <modelVersion>4.0.0</modelVersion> -  -  -  +  +  +  <parent> <groupId>org.nuiton</groupId> @@ -13,176 +14,183 @@ <version>2.1.5</version> </parent> - <artifactId>diswork</artifactId> - <version>0.0.1-SNAPSHOT</version> + <artifactId>diswork</artifactId> + <version>0.0.1-SNAPSHOT</version> - <modules> - <module>diswork-fs</module> - </modules> + <modules> + <module>diswork-fs</module> + </modules> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - <version>1.1.1</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <version>1.2.14</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>jgroups</groupId> - <artifactId>jgroups</artifactId> - <version>2.9.0.GA</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>1.4</version> - <type>jar</type> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>2.5</version> - <type>jar</type> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.nuiton</groupId> - <artifactId>nuiton-utils</artifactId> - <version>1.2.2</version> - <type>jar</type> - <scope>compile</scope> - </dependency> - -  - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.8.1</version> - <scope>test</scope> - </dependency> - </dependencies> - </dependencyManagement> -  -  -  + <dependencyManagement> + <dependencies> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.14</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>jgroups</groupId> + <artifactId>jgroups</artifactId> + <version>2.9.0.GA</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>1.4</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.5</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + <version>1.2.2</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>fr.inria.peerunit</groupId> + <artifactId>PeerUnit</artifactId> + <version>1.0</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.kth</groupId> + <artifactId>dks</artifactId> + <version>0.0.1</version> + </dependency> + <dependency> + <groupId>org.freepastry</groupId> + <artifactId>pastry</artifactId> + <version>2.1</version> + </dependency> + +  + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.8.1</version> + <scope>test</scope> + </dependency> + </dependencies> + </dependencyManagement> +  +  +  - <name>Diswork</name> - <description>Distributed computing application</description> - <inceptionYear>2010</inceptionYear> - <url>http://maven-site.nuiton.org/diswork</url> + <name>Diswork</name> + <description>Distributed computing application</description> + <inceptionYear>2010</inceptionYear> + <url>http://maven-site.nuiton.org/diswork</url> - <developers> - <developer> - <name>Brendan Le Ny</name> - <id>bleny</id> - <email>leny(a)codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - <developer> - <name>Benjamin Poussin</name> - <id>bpoussin</id> - <email>poussin(a)codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - <role>Debian packager</role> - </roles> - </developer> - <developer> - <name>Eric Chatellier</name> - <id>echatellier</id> - <email>chatellier(a)codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - <developer> - <name>Jean Couteau</name> - <id>jcouteau</id> - <email>couteau(a)codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - <developer> - <name>Tony Chemit</name> - <id>tchemit</id> - <email>chemit(a)codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - </developers> + <developers> + <developer> + <name>Brendan Le Ny</name> + <id>bleny</id> + <email>leny(a)codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + <developer> + <name>Benjamin Poussin</name> + <id>bpoussin</id> + <email>poussin(a)codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + <role>Debian packager</role> + </roles> + </developer> + <developer> + <name>Eric Chatellier</name> + <id>echatellier</id> + <email>chatellier(a)codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + <developer> + <name>Jean Couteau</name> + <id>jcouteau</id> + <email>couteau(a)codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + <developer> + <name>Tony Chemit</name> + <id>tchemit</id> + <email>chemit(a)codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + </developers> -  -  -  +  +  +  - <packaging>pom</packaging> + <packaging>pom</packaging> - <properties> -  - <projectId>diswork</projectId> - -  - <locales>en</locales> - </properties> + <properties> +  + <projectId>diswork</projectId> - <build> - +  + <locales>en</locales> + </properties> - <pluginManagement> - <plugins> - <plugin> - <artifactId>maven-site-plugin</artifactId> - <dependencies> - <dependency> - <groupId>org.nuiton.jrst</groupId> - <artifactId>doxia-module-jrst</artifactId> - <version>${jrst.version}</version> - </dependency> - </dependencies> - </plugin> - </plugins> - </pluginManagement> - </build> + <build> + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-site-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.nuiton.jrst</groupId> + <artifactId>doxia-module-jrst</artifactId> + <version>${jrst.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </pluginManagement> + </build> -  -  -  +  +  +  -  - <scm> - <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection> - <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection> - <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url> - </scm> +  + <scm> + <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection> + <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection> + <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url> + </scm> </project> Added: trunk/src/site/rst/diswork-fs/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-fs/how_it_works.rst (rev 0) +++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-02 13:08:02 UTC (rev 56) @@ -0,0 +1,125 @@ +How Diswork File System works +============================= + +Diswork File System uses a map to store data persistently. + +We illustrate the use of a file system by setting the content of "/", +the root directory as follow: + +* a directory named "my_folder" that contains: + + - a directory named "my_sub_folder" that contains: + + + a file name "my_deep_file.ext" + + + an empty folder named "my_empty_folder" + + - a file named "my_other_file.ext" + +* a file named "my_file.ext" (containing binary data: 010101) + +* a symbolic link named "my_link" which target the path + "/my_folder/my_sub_folder/my_deep_file.ext" + +Storage uses a Map to store data. + ++--------------+---------------------------------------------+ +| Key (String) | Value (byte[]) | ++==============+=============================================+ +| | "D:my_folder:ID1\n | +| | F:my_file.ext:ID2\n | +| "/" | L:my_link:ID3" | ++--------------+---------------------------------------------+ +| | "D:my_sub_folder:ID4\n | +| "ID1" | F:my_other_file.ext:ID5" | ++--------------+---------------------------------------------+ +| "ID2" | 010101 | ++--------------+---------------------------------------------+ +| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | ++--------------+---------------------------------------------+ +| | "F:my_deep_file:ID6\n | +| "ID4" | D:my_empty_folder:ID7" | ++--------------+---------------------------------------------+ +| "ID5" | 011010100111010... | ++--------------+---------------------------------------------+ +| "ID6" | 1100010101010110110010... | ++--------------+---------------------------------------------+ +| "ID7" | "" | ++--------------+---------------------------------------------+ + +An entry is, for example, "D:my_folder:ID1". It contains the type of the +entry (Directory, File or +Link), the name of the element, and an ID to be used as +a key on the map to get the actual content. Those three informations are +separated by ":" which is EntryUtil.ENTRY_SEPARATOR. + +A directory way have multiple entries. Entries of a directory are +separated by "\n" which is EntryUtil.ENTRIES_SEPARATOR. + +The above description shows the main principle used to store a tree +structure in a map. + +In fact, the storage of actual data is a bit more complex, due to +the need of splitting the data that may be too large (the value of "ID6" +may be a series of 100220 bytes for a 200 MiB file). +Actually, files content (like ID2, ID5, ID6) and directory content +values ("/", ID1, ID4) will be split and the map will look like that ( +note that only the files have been split for readability): + ++--------------+---------------------------------------------+ +| Key (String) | Value (byte[]) | ++==============+=============================================+ +| | "D:my_folder:ID1\n | +| | F:my_file.ext:ID2\n | +| "/" | L:my_link:ID3" | ++--------------+---------------------------------------------+ +| | "D:my_sub_folder:ID4\n | +| "ID1" | F:my_other_file.ext:ID5" | ++--------------+---------------------------------------------+ +| "ID2" | "6;ID8" | ++--------------+---------------------------------------------+ +| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | ++--------------+---------------------------------------------+ +| | "F:my_deep_file:ID6\n | +| "ID4" | D:my_empty_folder:ID7" | ++--------------+---------------------------------------------+ +| "ID5" | "14689;ID9" | ++--------------+---------------------------------------------+ +| "ID6" | "79874567;ID10;ID11;ID12" | ++--------------+---------------------------------------------+ +| "ID7" | "" | ++--------------+---------------------------------------------+ +| "ID8" | 010101 | ++--------------+---------------------------------------------+ +| "ID9" | 011010100111010... | ++--------------+---------------------------------------------+ +| "ID10" | 110001010101011... | ++--------------+---------------------------------------------+ +| "ID11" | 111010110101101... | ++--------------+---------------------------------------------+ +| "ID12" | 011101100111010... | ++--------------+---------------------------------------------+ + +Values of ID2, ID5, ID6 are no longer the file content. It's now a set +of meta-information information data called metablock. +A metablock is composed of the total size of the file followed by +an ordered lists of IDs of the different blocks composing the file. +Those informations are separated by ";" (see +{@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). + +In the above example: + +* "/my_file.ext" is a file which size is 6 bytes, it is + composed of one block available at ID8; + +* "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes) + which content is available at ID9; + +* "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its + size is 79874567. It has been split in three blocks: ID10, ID11 and + ID12. + +When reading and writing in storage, split is done transparently. When +reading, a Stream is returned: it loads data blocks after blocks +when needed inner class Storage.SplitBlocksInputStream}). +When writing, data are split in blocks of a maximum configurable size. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-02 12:56:07 UTC (rev 55) +++ trunk/src/site/site_en.xml 2010-06-02 13:08:02 UTC (rev 56) @@ -29,6 +29,7 @@ <menu name="diswork FS"> <item href="diswork-fs/history.html" name="Install"/> + <item href="diswork-fs/how_it_works.html" name="How it works"/> </menu> <menu ref="reports"/>

1 0

r55 - in trunk/diswork-fs: . src/main/java/org/nuiton/disworkfs src/main/java/org/nuiton/disworkfs/peerunit src/main/java/org/nuiton/disworkfs/storage src/main/resources src/test/java/org/nuiton/disworkfs src/test/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org June 2, 2010

June 2, 2010

Author: bleny Date: 2010-06-02 14:56:07 +0200 (Wed, 02 Jun 2010) New Revision: 55 Url: http://nuiton.org/repositories/revision/diswork/55 Log: implementation of DisworkMap with FreePastry ; more documentation Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/resources/freepastry.params trunk/diswork-fs/src/main/resources/tester.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/pom.xml 2010-06-02 12:56:07 UTC (rev 55) @@ -1,4 +1,5 @@ -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.nuiton</groupId> @@ -11,20 +12,20 @@ <packaging>jar</packaging> <name>disworkfs</name> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>2.0.2</version> - <configuration> - <source>1.6</source> - <target>1.6</target> - </configuration> - </plugin> - </plugins> - </build> - <dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.0.2</version> + <configuration> + <source>1.6</source> + <target>1.6</target> + </configuration> + </plugin> + </plugins> + </build> + <dependencies> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> @@ -41,12 +42,15 @@ <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> - +  <dependency> <groupId>junit</groupId> Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,221 @@ +package org.nuiton.disworkfs; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Random; + +import javax.swing.plaf.SliderUI; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.runner.Runner; + +public class Demo { + + private static final Log log = LogFactory.getLog(Demo.class); + + protected static DisworkFileSystem fileSystem; + + protected static class Producer implements Runnable { + + protected String waitingForJob = null; + protected Integer expectedResult = null; + + @Override + public void run() { + while (true) { + + try { + if (fileSystem.exists("/todo")) { + + if (waitingForJob == null) { + + // taking a random job + Random random = new Random(); + + waitingForJob = "/todo/job-" + random.nextInt(); + Integer randomInteger = random.nextInt(); + expectedResult = randomInteger + 1; + String job = randomInteger.toString(); + + InputStream source = IOUtils.toInputStream(job); + + + log.info("writing job " + waitingForJob + " (" + randomInteger + ")"); + + fileSystem.write(waitingForJob, source); + + source.close(); + + List<String> todos = fileSystem.readDirectory("/todo"); + + System.out.println("number of jobs : " + todos.size()); + + } else { + String resultPath = waitingForJob + ".result"; + if (fileSystem.exists(resultPath)) { + log.info("results has been published"); + + InputStream resultStream = fileSystem.read(resultPath); + + String resultString = IOUtils.toString(resultStream); + + Integer result = Integer.parseInt(resultString); + + log.info("result is " + result + "(expected : " + expectedResult + ")"); + + fileSystem.delete(resultPath); + fileSystem.delete(waitingForJob); + + waitingForJob = null; + expectedResult = null; + + } + + } + + + } else { + fileSystem.createDirectory("/todo"); + } + + } catch (IOException e) { + log.error(e); + } + + try { + Thread.sleep(15 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + + } + } + } + + protected static class Consumer implements Runnable { + + @Override + public void run() { + while (true) { + + try { + if (fileSystem.exists("/todo")) { + + List<String> todos = fileSystem.readDirectory("/todo"); + + if (todos.isEmpty()) { + log.info("nothing to do"); + } else { + // taking a random job + Random random = new Random(); + String todo = todos.get(random.nextInt(todos.size())); + + if (!todo.endsWith(".result")) { + String jobPath = "/todo/" + todo; + log.info("reading the job " + jobPath); + + InputStream in = fileSystem.read(jobPath); + + String operation = IOUtils.toString(in); + + // in.close(); + + log.info("operation to do " + operation); + + Integer i = Integer.parseInt(operation); + + i += 1; + + String result = i.toString(); + + log.info("result is " + result); + + InputStream source = IOUtils.toInputStream(result); + + fileSystem.write(jobPath + ".result", source); + + source.close(); + + } + } + + } else { + fileSystem.createDirectory("/todo"); + } + + } catch (IOException e) { + log.error(e); + } + + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + + } + } + } + + public static void printUsage() { + System.out.println( + "Usage :\n" + + "org.nuiton.disworkfs.Demo producer|consumer usePort [bootStrapIp bootStrapPort]" + ); + } + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + if (args.length == 2) { + + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + + config.setOption("diswork.fs.use_port", args[1]); + + fileSystem = new DisworkFileSystem(config); + + if ("producer".equals(args[0])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[0])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + + } else if (args.length == 4) { + + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + + config.setOption("diswork.fs.use_port", args[1]); + config.setOption("diswork.fs.bootstrap.ip", args[2]); + config.setOption("diswork.fs.bootstrap.port", args[3]); + + fileSystem = new DisworkFileSystem(config); + + if ("producer".equals(args[2])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[2])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + + } else { + printUsage(); + } + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-02 12:56:07 UTC (rev 55) @@ -11,13 +11,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; -import org.nuiton.disworkfs.storage.InMemoryMap; import org.nuiton.disworkfs.storage.Storage; /** Main class of Diswork File System, provide methods for all operations * You can use: * <ul> - * <li>{@link #createDirectory(String)} and {@link DisworkFileSystem#readDirectory(String)} + * <li>{@link #createDirectory(String)} and {@link #readDirectory(String)} * to create and browse directories</li> * <li>{@link #write(String, InputStream)} and {@link #read(String)} * to write a file and read it</li> @@ -44,13 +43,12 @@ /** constructor allowing to provide another configuration to use * - * @param disworkConfig the configuration + * @param config the configuration * @throws IOException */ - public DisworkFileSystem(DisworkFileSystemConfig disworkConfig) + public DisworkFileSystem(DisworkFileSystemConfig config) throws IOException { - storage = new Storage(disworkConfig, new InMemoryMap()); - storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + storage = new Storage(config); } /** tests the existence of a file/dir/link at a given path @@ -98,6 +96,9 @@ String id = EntryUtil.getIdFromEntry(entry); result = storage.getFile(id); } + + log.info("read " + path + " returns " + result.available() + " bytes"); + return result; } @@ -116,6 +117,7 @@ } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); + log.info("writing " + source.available() + " bytes at " + path); write(parent, name, source); } @@ -186,6 +188,7 @@ // store file before meta info storage.putDirectory(newDirectoryId, EntryUtil.newEmptyDirectoryContent()); + // update meta info directory storage.putDirectory(parentId, newContent); } else { @@ -329,18 +332,25 @@ entry = walk(target); } if (EntryUtil.isDirectory(entry)) { + result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); - String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR); - result = new ArrayList<String>(); - for (String elementEntry : entries) { - result.add(EntryUtil.getNameFromEntry(elementEntry)); + if (EntryUtil.newEmptyDirectoryContent().equals(content)) { + // directory is empty, add nothing + } else { + String[] entries = content.split( + EntryUtil.ENTRIES_SEPARATOR); + for (String elementEntry : entries) { + result.add(EntryUtil.getNameFromEntry(elementEntry)); + } } } else { throw new IOException(path + " is not a directory"); } } + log.info("readDirectory " + path + " returns " + result.size() + " results"); + return result; } @@ -418,7 +428,7 @@ log.info("in intermediate dir " + current + ", looking for " + p); // mise a jour de current - if (current.equals("/")) current = ""; + if (current.equals("/")) current = ""; // avoid "//path" next line current += "/" + p; String entry = EntryUtil.findEntryInDirectory(content, p); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-02 12:56:07 UTC (rev 55) @@ -15,12 +15,32 @@ * </dl> */ public class DisworkFileSystemConfig extends ApplicationConfig { - + public DisworkFileSystemConfig() { - setDefaultOption("blocks_size", "10485760"); // 10 MiB + setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB + setDefaultOption("diswork.fs.use_in_memory_map", "false"); + setDefaultOption("diswork.fs.use_port", "9001"); + setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); + setDefaultOption("diswork.fs.bootstrap.port", "9001"); } public int getBlockSize() { - return getOptionAsInt("blocks_size"); + return getOptionAsInt("diswork.fs.blocks_size"); } + + public boolean useInMemoryMap() { + return getOptionAsBoolean("diswork.fs.use_in_memory_map"); + } + + public Integer getUsedPort() { + return getOptionAsInt("diswork.fs.use_port"); + } + + public String getBootstrapIp() { + return getOption("diswork.fs.bootstrap.ip"); + } + + public Integer getBootstrapPort() { + return getOptionAsInt("diswork.fs.bootstrap.port"); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,10 +1,11 @@ /** * DisworkFS is a distributed file system. You can use it by instantiating - * {@link DisworkFileSystem} and use it to store directories and their content. + * {@link org.nuiton.disworkfs.DisworkFileSystem} and use it to store + * directories and their content. * * You can change the default DisworkFileSystem behavior by provide a - * {@link DisworkFileSystemConfig} instance at construction. If you don't - * provide one, a instance will be created + * {@link org.nuiton.disworkfs.DisworkFileSystemConfig} instance at + * construction. If you don't provide one, a instance will be created */ package org.nuiton.disworkfs; \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,93 @@ +package org.nuiton.disworkfs.peerunit; + +import static fr.inria.peerunit.test.assertion.Assert.assertTrue; +import static fr.inria.peerunit.test.assertion.Assert.assertEquals; +import static fr.inria.peerunit.test.assertion.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Arrays; + +import org.apache.commons.io.IOUtils; +import org.nuiton.disworkfs.DisworkFileSystem; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +import fr.inria.peerunit.TestCaseImpl; +import fr.inria.peerunit.parser.TestStep; + +public class DisworkFileSystemTest extends TestCaseImpl { + + protected Integer port = 9001; + + protected DisworkFileSystem fileSystem; + + protected byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; + + @TestStep(range="0",timeout=1000000, order = 0) + public void testConnect() throws Exception { + Integer myPort = port + getId(); + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + config.setOption("diswork.fs.use_port", myPort.toString()); + fileSystem = new DisworkFileSystem(config); + } + + @TestStep(range="1",timeout=1000000, order = 1) + public void testConnect1() throws Exception { + Integer myPort = port + getId(); + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + config.setOption("diswork.fs.use_port", myPort.toString()); + fileSystem = new DisworkFileSystem(config); + } + /* + @TestStep(range="0",timeout=1000000, order = 2) + public void testWrite() throws Exception { + InputStream source = new ByteArrayInputStream(bytes); + fileSystem.write("/myfile", source); + source.close(); + } + + @TestStep(range="1",timeout=1000000, order = 3) + public void testRead() throws Exception { + try { + InputStream source = fileSystem.read("/myfile"); + byte[] readResult = IOUtils.toByteArray(source); + source.close(); + boolean arraysContentEquals = Arrays.equals(bytes, readResult); + assertTrue(arraysContentEquals); + } catch (FileNotFoundException e) { + fail(); + } + } + */ + @TestStep(range="1",timeout=1000000, order = 4) + public void testCreateDir() throws Exception { + fileSystem.createDirectory("/mydir"); + fileSystem.createDirectory("/myseconddir"); + } + + @TestStep(range="0-1",timeout=1000000, order = 5) + public void fake() throws Exception { + + } + + /* + @TestStep(range="0-1",timeout=1000000, order = 5) + public void testCreateDir2() throws Exception { + assertTrue(fileSystem.exists("/mydir")); + assertTrue(fileSystem.exists("/myseconddir")); + } + */ + /* + @TestStep(range="1",timeout=1000000, order = 6) + public void testCreateSubDir() throws Exception { + fileSystem.createDirectory("/mydir/mysubdir"); + } + + @TestStep(range="1",timeout=1000000, order = 7) + public void testReadDir() throws Exception { + assertTrue(fileSystem.exists("/mydir/mysubdir")); + assertEquals(1, fileSystem.readDirectory("/mydir").size()); + } + */ +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,6 @@ +package org.nuiton.disworkfs.storage; + +import java.io.Closeable; +import java.util.Map; + +public interface DisworkMap extends Map<String, byte[]>, Closeable {} Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java (from rev 52, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,32 @@ +package org.nuiton.disworkfs.storage; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** An in-memory implementation of a Map + * This class is a map where put make a copy of the value. It is used for + * stubbing purpose. It emulates the behavior of a DHT and thus can be used + * by {@link Storage} to store data and blocks. + */ +public class InMemoryDisworkMap extends HashMap<String, byte[]> + implements DisworkMap { + + private final Log log = LogFactory.getLog(InMemoryDisworkMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } + + @Override + public void close() throws IOException { + // nothing to do + } +} \ No newline at end of file Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,25 +0,0 @@ -package org.nuiton.disworkfs.storage; - -import java.util.Arrays; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** An in-memory implementation of a Map - * This class is a map where put make a copy of the value. It is used for - * stubbing purpose. It emulates the behavior of a DHT and thus can be used - * by {@link Storage} to store data and blocks. - */ -public class InMemoryMap extends HashMap<String, byte[]> { - - private final Log log = LogFactory.getLog(InMemoryMap.class); - - @Override - public byte[] put(String key, byte[] value) { - log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); - byte[] valueCopy = value.clone(); - log.info("putting " + valueCopy.length + " bytes"); - return super.put(key, valueCopy); - } -} \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,363 @@ +package org.nuiton.disworkfs.storage; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +import rice.Continuation; +import rice.environment.Environment; +import rice.p2p.commonapi.Id; +import rice.p2p.past.ContentHashPastContent; +import rice.p2p.past.PastContent; +import rice.p2p.past.PastImpl; +import rice.pastry.NodeIdFactory; +import rice.pastry.PastryNode; +import rice.pastry.PastryNodeFactory; +import rice.pastry.commonapi.PastryIdFactory; +import rice.pastry.socket.SocketPastryNodeFactory; +import rice.pastry.standard.RandomNodeIdFactory; +import rice.persistence.EmptyCache; +import rice.persistence.MemoryStorage; +import rice.persistence.StorageManagerImpl; + +/** + * An implementation of {@link DisworkMap} using FreePastry. + * + * @see <a href="https://trac.freepastry.org/">FreePastry site</a> + * + */ +public class PastryDisworkMap implements DisworkMap { + + private final Log log = LogFactory.getLog(PastryDisworkMap.class); + + protected static class ByteArrayPastContent extends ContentHashPastContent { + + private static final long serialVersionUID = 7584834640788361309L; + + byte[] content; + + public ByteArrayPastContent(Id id, byte[] content) { + super(id); + this.content = content; + } + + public String toString() { + return "ByteArrayPastContent (" + content.length + " bytes)"; + } + } + + protected abstract class MyContinuation<R, E extends Exception> implements Continuation<R, Exception> { + + protected boolean finished = false; + + public boolean isFinished() { + return finished; + } + + public void waitUntilFinised() { + while(!finished) { + try { + Thread.sleep(1500); + log.info("waiting for response"); + } catch (InterruptedException e) { + // TODO 20100527 bleny stub + log.info("exception catch", e); + e.printStackTrace(); + } + } + } + + @Override + public void receiveException(Exception exception) { + log.error("exception received : " + exception); + finished = true; + } + + @Override + public void receiveResult(R result) { + finished = true; + } + + } + + protected PastImpl past; + + protected PastryIdFactory pastryIdFactory; + + public PastryDisworkMap(DisworkFileSystemConfig config) throws IOException { + + Environment env = new Environment(); + + // disable the UPnP setting (in case you are testing this on a NATted LAN) + // env.getParameters().setString("nat_search_policy", "never"); + + // the port to use locally + int bindport = config.getUsedPort(); + + InetAddress bootaddr = InetAddress.getByName(config.getBootstrapIp()); + int bootport = config.getBootstrapPort(); + + InetSocketAddress bootaddress = new InetSocketAddress(bootaddr, bootport); + + log.info("boot address : " + bootaddress); + + NodeIdFactory nidFactory = new RandomNodeIdFactory(env); + + // construct the PastryNodeFactory, this is how we use rice.pastry.socket + PastryNodeFactory factory = null; + + // FIXME 20100602 bleny this is a work-around to deal with time out on + // connect exceptions that occurs "sometimes" + int numberOfTry = 0; + while(factory == null && numberOfTry <= 10) { + try { + factory = new SocketPastryNodeFactory(nidFactory, bindport, env); + } catch (java.net.ConnectException e) { + numberOfTry += 1; + } + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + log.info("exception catch", e); + e.printStackTrace(); + } + } + if (factory == null) { + throw new IOException("unable to connect"); + } + + // construct a node, but this does not cause it to boot + PastryNode node = factory.newNode(); + + // in later tutorials, we will register applications before calling boot + node.boot(bootaddress); + + // the node may require sending several messages to fully boot into the ring + synchronized(node) { + while(!node.isReady() && !node.joinFailed()) { + // delay so we don't busy-wait + try { + node.wait(500); + } catch (InterruptedException e) { + // TODO 20100528 bleny Auto-generated catch block + log.info("exception catch", e); + e.printStackTrace(); + } + + // abort if can't join + if (node.joinFailed()) { + throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason()); + } + } + } + + log.info("finished creating new node " + node); + + pastryIdFactory = new PastryIdFactory(env); + + rice.persistence.Storage storage = new MemoryStorage(pastryIdFactory); + past = new PastImpl( + node, + new StorageManagerImpl( + pastryIdFactory, + storage, + // Using a cache do not permit to remove while putting + /* + new LRUCache( + new MemoryStorage(pastryIdFactory), + 512 * 1024, + node.getEnvironment() + ) + */ + new EmptyCache(pastryIdFactory) + ), + 0, // replication factor + "" + ); + + + } + + protected class ContainsKeyContinuation extends MyContinuation<Boolean, Exception> { + + public Boolean result = null; + + @Override + public void receiveResult(Boolean result) { + log.info("contains key result received : " + result); + this.result = result; + finished = true; + } + } + + @Override + public boolean containsKey(Object key) { + Id id = pastryIdFactory.buildId((String) key); + ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation(); + past.existsInOverlay(id, containsKeyContinuation); + containsKeyContinuation.waitUntilFinised(); + boolean result = containsKeyContinuation.result; + log.info("containsKey " + key + " (id = " + id + ") returns " + result); + return result; + } + + protected class GetContinuation extends MyContinuation<PastContent, Exception> { + + public byte[] result = null; + + @Override + public void receiveResult(PastContent result) { + log.info("get result received " + result); + if (result != null) { + this.result = ((ByteArrayPastContent) result).content; + } + finished = true; + } + } + + protected byte[] atomicGet(Object key) { + Id id = pastryIdFactory.buildId((String) key); + GetContinuation getContinuation = new GetContinuation(); + past.lookup(id, getContinuation); + getContinuation.waitUntilFinised(); + return getContinuation.result; + } + + @Override + public byte[] get(Object key) { + byte[] result = atomicGet(key); + if (result == null) { + log.info("get " + key + " returns null"); + } else { + log.info("get " + key + " returns " + result.length + " bytes"); + } + return result; + } + + protected class PutContinuation extends MyContinuation<Boolean[], Exception> { + @Override + public void receiveResult(Boolean[] result) { + log.info("insert result received : " + Arrays.toString(result)); + finished = true; + } + } + + protected void atomicPut(String key, byte[] value) { + Id id = pastryIdFactory.buildId(key); + PastContent pastContent = new ByteArrayPastContent(id, value); + PutContinuation putContinuation = new PutContinuation(); + past.insert(pastContent, putContinuation); + putContinuation.waitUntilFinised(); + } + + @Override + public byte[] put(String key, byte[] value) { + byte[] previousValue = atomicGet(key); + if (previousValue != null) { + atomicRemove(key); + } + atomicPut(key, value); + log.info("put " + value.length + " bytes at " + key ); + return previousValue; + } + + protected class RemoveContinuation extends MyContinuation<Boolean, Exception> { + + public Boolean result = null; + + @Override + public void receiveResult(Boolean result) { + log.info("remove result received : " + result); + this.result = result; + finished = true; + } + } + + protected void atomicRemove(Object key) { + Id id = pastryIdFactory.buildId((String) key); + RemoveContinuation removeContinuation = new RemoveContinuation(); + past.remove(id, removeContinuation); + removeContinuation.waitUntilFinised(); + log.info("atomic remove " + key + " has returned " + removeContinuation.result); + } + + @Override + public byte[] remove(Object key) { + byte[] previousValue = atomicGet(key); + if (previousValue == null) { + log.info("remove " + key + " returns null"); + } else { + atomicRemove(key); + log.info("remove " + key + " returns " + previousValue.length + " bytes"); + } + return previousValue; + } + + @Override + public void close() throws IOException { + // TODO 20100528 bleny should close Pastry and close all sockets + // throw new UnsupportedOperationException("not yet implemented"); + /* + ServerSocket s = new ServerSocket(9001); + s.getChannel().close(); + s.close(); + */ + } + + @Deprecated + @Override + public void putAll(Map<? extends String, ? extends byte[]> arg0) { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public int size() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Collection<byte[]> values() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public boolean isEmpty() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Set<String> keySet() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public boolean containsValue(Object arg0) { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Set<java.util.Map.Entry<String, byte[]>> entrySet() { + throw new UnsupportedOperationException(); + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-02 12:56:07 UTC (rev 55) @@ -15,7 +15,11 @@ import org.nuiton.disworkfs.DisworkFileSystemConfig; /** + * This class is the middle layer between the File System operations and + * the Map<String, byte[]>. It offers put and get on directories, files + * and links and deal with splitting consideration and concurrency * + * It needs a place to store data called a {@link DisworkMap}. */ public class Storage implements Closeable { @@ -104,13 +108,23 @@ } - protected Map<String, byte[]> map; + protected DisworkMap map; protected DisworkFileSystemConfig disworkConfig; - public Storage(DisworkFileSystemConfig disworkConfig, Map<String, byte[]> map) { - this.map = map; + public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { this.disworkConfig = disworkConfig; + if (disworkConfig.useInMemoryMap()) { + log.info("using in-memory map"); + this.map = new InMemoryDisworkMap(); + putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + } else { + log.info("using Pastry map"); + this.map = new PastryDisworkMap(disworkConfig); + if (! map.containsKey("/")) { + putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + } + } } /** @@ -189,30 +203,36 @@ // trying to acquire lock Long currentDate = System.currentTimeMillis(); - byte[] lock = map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString())); + log.info("trying to acquire a lock on " + key); + byte[] lock = map.put(lockKey, + EntryUtil.stringToBytes(currentDate.toString())); if (lock != null) { // file is locked, check date Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock)); - - if (currentDate - currentLock > LOCK_VALID_TIME) { - // this lock is out-dated, let's erase it + Long lockAge = currentDate - currentLock; + if (lockAge > LOCK_VALID_TIME) { + log.info("lock is out-dated (" + lockAge + " ms old)"); + // this lock is out-dated, let's erase all data String obsoleteMetaBlock = EntryUtil.bytesToString(map.get(newDataKey)); if (obsoleteMetaBlock != null) { String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); + log.info("removing " + obsoleteBlocksIds.length + " old blocks"); for (String obsoleteBlockId : obsoleteBlocksIds) { map.remove(obsoleteBlockId); } } } else { - // file is currently written, stopping operation + log.info(key + " is currently written, stopping operation"); map.put(lockKey, lock); throw new ConcurrentModificationException("can't write due to concurrency"); } } + log.info("lock on " + key + " acquired"); + // here, we know we can write or an exception // would have been thrown earlier String blocksIds = ""; @@ -258,6 +278,8 @@ } map.put(key, map.get(newDataKey)); + + log.info("release lock " + lockKey); map.remove(lockKey); } @@ -283,6 +305,7 @@ @Override public void close() throws IOException { clean(); + map.close(); } } \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,7 +1,8 @@ /** * * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a - * way to persistently store data. This is done by the {@link Storage} class + * way to persistently store data. This is done by the + * {@link org.nuiton.disworkfs.storage.Storage} class * which permit to store different type of data. * * @@ -82,12 +83,14 @@ * entry (Directory, File or * Link), the name of the element, and an ID to be used as * a key on the map to get the actual content. Those three informations are - * separated by ":" which is {@link EntryUtil#ENTRY_SEPARATOR}. + * separated by ":" which is + * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRY_SEPARATOR}. * * * * A directory way have multiple entries. Entries of a directory are - * separated by "\n" which is {@link EntryUtil#ENTRIES_SEPARATOR}. + * separated by "\n" which is + * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRIES_SEPARATOR}. * * * The above description shows the main principle used to store a tree @@ -171,7 +174,7 @@ * A metablock is composed of the total size of the file followed by * an ordered lists of IDs of the different blocks composing the file. * Those informations are separated by ";" (see - * {@link EntryUtil#BLOCKIDS_SEPARATOR}). + * {@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). * * * In the above example: @@ -194,9 +197,10 @@ * * When reading and writing in storage, split is done transparently. When * reading, a Stream is returned: it loads data blocks after blocks - * when needed (see {@link Storage.SplitBlocksInputStream}). When writing, - * data are split in blocks of a maximum configurable size - * (see {@link org.nuiton.disworkfs.DisworkConfig}). + * when needed (see + * {@link org.nuiton.disworkfs.storage.Storage.SplitBlocksInputStream}). + * When writing, data are split in blocks of a maximum configurable size + * (see {@link org.nuiton.disworkfs.DisworkFileSystemConfig}). * */ Added: trunk/diswork-fs/src/main/resources/freepastry.params =================================================================== --- trunk/diswork-fs/src/main/resources/freepastry.params (rev 0) +++ trunk/diswork-fs/src/main/resources/freepastry.params 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,404 @@ +# this file holds the default values for pastry and it's applications +# you do not need to modify the default.params file to override these values +# instead you can use your own params file to set values to override the +# defaults. You can specify this file by constructing your +# rice.environment.Environment() with the filename you wish to use +# typically, you will want to be able to pass this file name from the command +# line + +# max number of handles stored per routing table entry +pastry_rtMax = 1 +pastry_rtBaseBitLength = 4 + +# leafset size +pastry_lSetSize = 24 + +# maintenance frequencies +pastry_leafSetMaintFreq = 60 +pastry_routeSetMaintFreq = 900 + +# drop the message if pastry is not ready +pastry_messageDispatch_bufferIfNotReady = false + +# number of messages to buffer while an app hasn't yet been registered +pastry_messageDispatch_bufferSize = 32 + +# FP 2.1 uses the new transport layer +transport_wire_datagram_receive_buffer_size = 131072 +transport_wire_datagram_send_buffer_size = 65536 +transport_epoch_max_num_addresses = 2 +transport_sr_max_num_hops = 5 + +# proximity neighbor selection +transport_use_pns = true + +# number of rows in the routing table to consider during PNS +# valid values are ALL, or a number +pns_num_rows_to_use = 10 + +# commonapi testing parameters + +# direct or socket +commonapi_testing_exit_on_failure = true +commonapi_testing_protocol = direct +commonapi_testing_startPort = 5009 +commonapi_testing_num_nodes = 10 +# set this to specify the bootstrap node +#commonapi_testing_bootstrap = localhost:5009 + +# random number generator's seed, "CLOCK" uses the current clock time +random_seed = CLOCK + +# sphere, euclidean or gt-itm +direct_simulator_topology = sphere +# -1 starts the simulation with the current time +direct_simulator_start_time = -1 +#pastry_direct_use_own_random = true +#pastry_periodic_leafset_protocol_use_own_random = true +pastry_direct_gtitm_matrix_file=GNPINPUT +# the number of stubs in your network +pastry_direct_gtitm_max_overlay_size=1000 +# the number of virtual nodes at each stub: this allows you to simulate multiple "LANs" and allows cheeper scaling +pastry_direct_gtitm_nodes_per_stub=1 +# the factor to multiply your file by to reach millis. Set this to 0.001 if your file is in microseconds. Set this to 1000 if your file is in seconds. +pastry_direct_gtitm_delay_factor=1.0 +#millis of the maximum network delay for the generated network topologies +pastry_direct_max_diameter=200 +pastry_direct_min_delay=2 +#setting this to false will use the old protocols which are about 200 times as fast, but may cause routing inconsistency in a real network. Probably won't in a simulator because it will never be incorrect about liveness +pastry_direct_guarantee_consistency=true + +# rice.pastry.socket parameters +# tells the factory you intend to use multiple nodes +# this causes the logger to prepend all entries with the nodeid +pastry_factory_multipleNodes = true +pastry_factory_selectorPerNode = false +pastry_factory_processorPerNode = false +# number of bootstap nodehandles to fetch in parallel +pastry_factory_bootsInParallel = 1 + +# the maximum size of a message +pastry_socket_reader_selector_deserialization_max_size = 1000000 +# the maximum number of outgoing messages to queue when a socket is slower than the number of messages you are queuing +pastry_socket_writer_max_queue_length = 30 +pastry_socket_writer_max_msg_size = 20480 +pastry_socket_repeater_buffer_size = 65536 +pastry_socket_pingmanager_smallPings=true +pastry_socket_pingmanager_datagram_receive_buffer_size = 131072 +pastry_socket_pingmanager_datagram_send_buffer_size = 65536 +# the time before it will retry a route that was already found dead +pastry_socket_srm_check_dead_throttle = 300000 +pastry_socket_srm_proximity_timeout = 3600000 +pastry_socket_srm_ping_throttle = 30000 +pastry_socket_srm_default_rto = 3000 +pastry_socket_srm_rto_ubound = 10000 +pastry_socket_srm_rto_lbound = 50 +pastry_socket_srm_gain_h = 0.25 +pastry_socket_srm_gain_g = 0.125 +pastry_socket_scm_max_open_sockets = 300 +pastry_socket_scm_max_open_source_routes = 30 +# the maximum number of source routes to attempt, setting this to 0 will +# effectively eliminate source route attempts +# setting higher than the leafset does no good, it will be bounded by the leafset +# a larger number tries more source routes, which could give you a more accurate +# determination, however, is more likely to lead to congestion collapse +pastry_socket_srm_num_source_route_attempts = 8 +pastry_socket_scm_socket_buffer_size = 32768 +# this parameter is multiplied by the exponential backoff when doing a liveness check so the first will be 800, then 1600, then 3200 etc... +pastry_socket_scm_ping_delay = 800 +# adds some fuzziness to the pings to help prevent congestion collapse, so this will make the ping be advanced or delayed by this factor +pastry_socket_scm_ping_jitter = 0.1 +# how many pings until we call the node faulty +pastry_socket_scm_num_ping_tries = 5 +pastry_socket_scm_write_wait_time = 30000 +pastry_socket_scm_backoff_initial = 250 +pastry_socket_scm_backoff_limit = 5 +pastry_socket_pingmanager_testSourceRouting = false +pastry_socket_increment_port_after_construction = true +# if you want to allow connection to 127.0.0.1, set this to true +pastry_socket_allow_loopback = false +# these params will be used if the computer attempts to bind to the loopback address, they will open a socket to this address/port to identify which network adapter to bind to +pastry_socket_known_network_address = yahoo.com +pastry_socket_known_network_address_port = 80 +pastry_socket_use_own_random = true +pastry_socket_random_seed = clock +# force the node to be a seed node +rice_socket_seed = false + +# the parameter simulates some nodes being firewalled, base on rendezvous_test_num_firewalled +rendezvous_test_firewall = false +# probabilistic fraction of firewalled nodes +rendezvous_test_num_firewalled = 0.3 +# don't firewall the first node, useful for testing +rendezvous_test_makes_bootstrap = false + +# FP 2.1 uses the new transport layer +transport_wire_datagram_receive_buffer_size = 131072 +transport_wire_datagram_send_buffer_size = 65536 + +# NAT/UPnP settings +nat_network_prefixes = 127.0.0.1;10.;192.168. +# Enable and set this if you have already set up port forwarding and know the external address +#external_address = 123.45.67.89:1234 +#enable this if you set up port forwarding (on the same port), but you don't +#know the external address and you don't have UPnP enabled +#this is useful for a firwall w/o UPnP support, and your IP address isn't static +probe_for_external_address = false +# values how to probe +pastry_proxy_connectivity_timeout = 15000 +pastry_proxy_connectivity_tries = 3 +# possible values: always, never, prefix (prefix is if the localAddress matches any of the nat_network_prefixes +# whether to search for a nat using UPnP (default: prefix) +nat_search_policy = prefix +# whether to verify connectivity (default: boot) +firewall_test_policy = boot +# policy for setting port forwarding the state of the firewall if there is already a conflicting rule: overwrite, fail (throw exception), change (use different port) +# you may want to set this to overwrite or fail on the bootstrap nodes, but most freepastry applications can run on any available port, so the default is change +nat_state_policy = change +# the name of the application in the firewall, set this if you want your application to have a more specific name +nat_app_name = freepastry +# how long to wait for responses from the firewall, in millis +nat_discovery_timeout = 5000 +# how many searches to try to find a free firewall port +nat_find_port_max_tries = 10 +# uncomment this to use UPnP NAT port forwarding, you need to include in the classpath: commons-jxpath-1.1.jar:commons-logging.jar:sbbi-upnplib-xxx.jar +nat_handler_class = rice.pastry.socket.nat.sbbi.SBBINatHandler +# hairpinning: +# default "prefix" requires more bandwidth if you are behind a NAT. It enables multiple IP +# addresses in the NodeHandle if you are behind a NAT. These are usually the internet routable address, +# and the LAN address (usually 192.168.x.x) +# you can set this to never if any of the following conditions hold: +# a) you are the only FreePastry node behind this address +# b) you firewall supports hairpinning see +# http://scm.sipfoundry.org/rep/ietf-drafts/behave/draft-ietf-behave-nat-udp-… +nat_nodehandle_multiaddress = prefix + +# if we are not scheduled for time on cpu in this time, we setReady(false) +# otherwise there could be message inconsistency, because +# neighbors may believe us to be dead. Note that it is critical +# to consider the amount of time it takes the transport layer to find a +# node faulty before setting this parameter, this parameter should be +# less than the minimum time required to find a node faulty +pastry_protocol_consistentJoin_max_time_to_be_scheduled = 15000 + +# in case messages are dropped or something, how often it will retry to +# send the consistent join message, to get verification from the entire +# leafset +pastry_protocol_consistentJoin_retry_interval = 30000 +# parameter to control how long dead nodes are retained in the "failed set" in +# CJP (see ConsistentJoinProtocol ctor) (15 minutes) +pastry_protocol_consistentJoin_failedRetentionTime = 900000 +# how often to cleanup the failed set (5 mins) (see ConsistentJoinProtocol ctor) +pastry_protocol_consistentJoin_cleanup_interval = 300000 +# the maximum number of entries to send in the failed set, only sends the most +recent detected failures (see ConsistentJoinProtocol ctor) +pastry_protocol_consistentJoin_maxFailedToSend = 20 + +# how often we send/expect to be sent updates +pastry_protocol_periodicLeafSet_ping_neighbor_period = 20000 +pastry_protocol_periodicLeafSet_lease_period = 30000 + +# what the grace period is to receive a periodic update, before checking +# liveness +pastry_protocol_periodicLeafSet_request_lease_throttle = 10000 + +# how many entries are kept in the partition handler's table +partition_handler_max_history_size=20 +# how long entries in the partition handler's table are kept +# 90 minutes +partition_handler_max_history_age=5400000 +# what fraction of the time a bootstrap host is checked +partition_handler_bootstrap_check_rate=0.05 +# how often to run the partition handler +# 5 minutes +partition_handler_check_interval=300000 + +# the version number of the RouteMessage to transmit (it can receive anything that it knows how to) +# this is useful if you need to migrate an older ring +# you can change this value in realtime, so, you can start at 0 and issue a command to update it to 1 +pastry_protocol_router_routeMsgVersion = 1 + +# should usually be equal to the pastry_rtBaseBitLength +p2p_splitStream_stripeBaseBitLength = 4 +p2p_splitStream_policy_default_maximum_children = 24 +p2p_splitStream_stripe_max_failed_subscription = 5 +p2p_splitStream_stripe_max_failed_subscription_retry_delay = 1000 + +#multiring +p2p_multiring_base = 2 + +#past +p2p_past_messageTimeout = 30000 +p2p_past_successfulInsertThreshold = 0.5 + +#replication + +# fetch delay is the delay between fetching successive keys +p2p_replication_manager_fetch_delay = 500 +# the timeout delay is how long we take before we time out fetching a key +p2p_replication_manager_timeout_delay = 20000 +# this is the number of keys to delete when we detect a change in the replica set +p2p_replication_manager_num_delete_at_once = 100 +# this is how often replication will wake up and do maintainence; 10 mins +p2p_replication_maintenance_interval = 600000 +# the maximum number of keys replication will try to exchange in a maintainence message +p2p_replication_max_keys_in_message = 1000 + +#scribe +p2p_scribe_maintenance_interval = 180000 +#time for a subscribe fail to be thrown (in millis) +p2p_scribe_message_timeout = 15000 + +#util +p2p_util_encryptedOutputStream_buffer = 32678 + +#aggregation +p2p_aggregation_logStatistics = true +p2p_aggregation_flushDelayAfterJoin = 30000 +#5 MINS +p2p_aggregation_flushStressInterval = 300000 +#5 MINS +p2p_aggregation_flushInterval = 300000 +#1024*1024 +p2p_aggregation_maxAggregateSize = 1048576 +p2p_aggregation_maxObjectsInAggregate = 25 +p2p_aggregation_maxAggregatesPerRun = 2 +p2p_aggregation_addMissingAfterRefresh = true +p2p_aggregation_maxReaggregationPerRefresh = 100 +p2p_aggregation_nominalReferenceCount = 2 +p2p_aggregation_maxPointersPerAggregate = 100 +#14 DAYS +p2p_aggregation_pointerArrayLifetime = 1209600000 +#1 DAY +p2p_aggregation_aggregateGracePeriod = 86400000 +#15 MINS +p2p_aggregation_aggrRefreshInterval = 900000 +p2p_aggregation_aggrRefreshDelayAfterJoin = 70000 +#3 DAYS +p2p_aggregation_expirationRenewThreshold = 259200000 +p2p_aggregation_monitorEnabled = false +#15 MINS +p2p_aggregation_monitorRefreshInterval = 900000 +#5 MINS +p2p_aggregation_consolidationDelayAfterJoin = 300000 +#15 MINS +p2p_aggregation_consolidationInterval = 900000 +#14 DAYS +p2p_aggregation_consolidationThreshold = 1209600000 +p2p_aggregation_consolidationMinObjectsInAggregate = 20 +p2p_aggregation_consolidationMinComponentsAlive = 0.8 +p2p_aggregation_reconstructionMaxConcurrentLookups = 10 +p2p_aggregation_aggregateLogEnabled = true +#1 HOUR +p2p_aggregation_statsGranularity = 3600000 +#3 WEEKS +p2p_aggregation_statsRange = 1814400000 +p2p_aggregation_statsInterval = 60000 +p2p_aggregation_jitterRange = 0.1 + +# glacier +p2p_glacier_logStatistics = true +p2p_glacier_faultInjectionEnabled = false +p2p_glacier_insertTimeout = 30000 +p2p_glacier_minFragmentsAfterInsert = 3.0 +p2p_glacier_refreshTimeout = 30000 +p2p_glacier_expireNeighborsDelayAfterJoin = 30000 +#5 MINS +p2p_glacier_expireNeighborsInterval = 300000 +#5 DAYS +p2p_glacier_neighborTimeout = 432000000 +p2p_glacier_syncDelayAfterJoin = 30000 +#5 MINS +p2p_glacier_syncMinRemainingLifetime = 300000 +#insertTimeout +p2p_glacier_syncMinQuietTime = 30000 +p2p_glacier_syncBloomFilterNumHashes = 3 +p2p_glacier_syncBloomFilterBitsPerKey = 4 +p2p_glacier_syncPartnersPerTrial = 1 +#1 HOUR +p2p_glacier_syncInterval = 3600000 +#3 MINUTES +p2p_glacier_syncRetryInterval = 180000 +p2p_glacier_syncMaxFragments = 100 +p2p_glacier_fragmentRequestMaxAttempts = 0 +p2p_glacier_fragmentRequestTimeoutDefault = 10000 +p2p_glacier_fragmentRequestTimeoutMin = 10000 +p2p_glacier_fragmentRequestTimeoutMax = 60000 +p2p_glacier_fragmentRequestTimeoutDecrement = 1000 +p2p_glacier_manifestRequestTimeout = 10000 +p2p_glacier_manifestRequestInitialBurst = 3 +p2p_glacier_manifestRequestRetryBurst = 5 +p2p_glacier_manifestAggregationFactor = 5 +#3 MINUTES +p2p_glacier_overallRestoreTimeout = 180000 +p2p_glacier_handoffDelayAfterJoin = 45000 +#4 MINUTES +p2p_glacier_handoffInterval = 240000 +p2p_glacier_handoffMaxFragments = 10 +#10 MINUTES +p2p_glacier_garbageCollectionInterval = 600000 +p2p_glacier_garbageCollectionMaxFragmentsPerRun = 100 +#10 MINUTES +p2p_glacier_localScanInterval = 600000 +p2p_glacier_localScanMaxFragmentsPerRun = 20 +p2p_glacier_restoreMaxRequestFactor = 4.0 +p2p_glacier_restoreMaxBoosts = 2 +p2p_glacier_rateLimitedCheckInterval = 30000 +p2p_glacier_rateLimitedRequestsPerSecond = 3 +p2p_glacier_enableBulkRefresh = true +p2p_glacier_bulkRefreshProbeInterval = 3000 +p2p_glacier_bulkRefreshMaxProbeFactor = 3.0 +p2p_glacier_bulkRefreshManifestInterval = 30000 +p2p_glacier_bulkRefreshManifestAggregationFactor = 20 +p2p_glacier_bulkRefreshPatchAggregationFactor = 50 +#3 MINUTES +p2p_glacier_bulkRefreshPatchInterval = 180000 +p2p_glacier_bulkRefreshPatchRetries = 2 +p2p_glacier_bucketTokensPerSecond = 100000 +p2p_glacier_bucketMaxBurstSize = 200000 +p2p_glacier_jitterRange = 0.1 +#1 MINUTE +p2p_glacier_statisticsReportInterval = 60000 +p2p_glacier_maxActiveRestores = 3 + +#transport layer testing params +org.mpisws.p2p.testing.transportlayer.replay.Recorder_printlog = true + +# logging +#default log level +loglevel = WARNING + +#example of enabling logging on the endpoint: +#rice.p2p.scribe@ScribeRegrTest-endpoint_loglevel = INFO + +logging_packageOnly = true + +logging_date_format = yyyyMMdd.HHmmss.SSS +logging_enable=true + +# 24 hours +log_rotate_interval = 86400000 +# the name of the active log file, and the filename prefix of rotated log +log_rotate_filename = freepastry.log +# the format of the date for the rotating log +log_rotating_date_format = yyyyMMdd.HHmmss.SSS + +# true will tell the environment to ues the FileLogManager +environment_logToFile = false +# the prefix for the log files (otherwise will be named after the nodeId) +fileLogManager_filePrefix = +# the suffix for the log files +fileLogManager_fileSuffix = .log +# wether to keep the line prefix (declaring the node id) for each line of the log +fileLogManager_keepLinePrefix = false +fileLogManager_multipleFiles = true +fileLogManager_defaultFileName = main + +# false = append true = overwrite +fileLogManager_overwrite_existing_log_file = false + +# the amount of time the LookupService tutorial app will wait before timing out +# in milliseconds, default is 30 seconds +lookup_service.timeout = 30000 +# how long to wait before the first retry +lookup_service.firstTimeout = 500 Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-02 12:56:07 UTC (rev 55) @@ -5,4 +5,5 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=WARN \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=INFO +log4j.logger.org.nuiton.disworkfs.Demo=INFO \ No newline at end of file Added: trunk/diswork-fs/src/main/resources/tester.properties =================================================================== --- trunk/diswork-fs/src/main/resources/tester.properties (rev 0) +++ trunk/diswork-fs/src/main/resources/tester.properties 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,11 @@ +# IP where the coordinator will execute +tester.server=127.0.0.1 + +# path and the name for the peerunit logfile +tester.logfile=peerunit.log + +# path to store the logfiles of the peers +tester.logfolder=/tmp/peerunit + +# number of peer that execute the test +tester.peers=2 Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,19 @@ +package org.nuiton.disworkfs; + +import org.junit.Before; + +public class DisworkFileSystemInMemoryTest extends DisworkFileSystemTest { + + /** + * this code executed after {@link DisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); + disworkConfig.setOption("diswork.fs.use_in_memory_map", "true"); + fileSystem = new DisworkFileSystem(disworkConfig); + } + +} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,82 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Before; +import org.junit.Test; + +public class DisworkFileSystemPastryTest extends DisworkFileSystemTest { + + /** + * this code executed after {@link DisworkFileSystemTest#setUp()} + * @throws Exception + */ + + private static final Log log = LogFactory.getLog(DisworkFileSystemPastryTest.class); + + protected Integer bootstrapPort; + + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = Util.newDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + + } + + /* + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + log.info("----------------------trying to create second directory"); + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + */ + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -4,6 +4,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.File; @@ -11,6 +12,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ConcurrentModificationException; import java.util.List; import java.util.Random; @@ -22,7 +24,7 @@ import org.nuiton.util.FileUtil; -public class DisworkFileSystemTest { +public abstract class DisworkFileSystemTest { /** * a place to store files for the test it's a subdirectory of the OS temp @@ -66,9 +68,6 @@ File randomFile = new File(randomFilePath); FileUtils.writeByteArrayToFile(randomFile, randomBytes); - // finally, initiate the fileSystem - DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); - fileSystem = new DisworkFileSystem(disworkConfig); } @After @@ -311,6 +310,18 @@ // ... and only those assertEquals(3, lsResult.size()); + + lsResult = fileSystem.readDirectory("/"); + assertEquals(1, lsResult.size()); } - + + @Test + public void testConcurrency() throws Exception { + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + } } Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,159 +0,0 @@ -/* *##% - * Copyright (c) 2010 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.nuiton.disworkfs; - - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; -import org.nuiton.disworkfs.storage.EntryUtil; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class EntryUtilTest { - - static protected String directoryEntry = - EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "mydir" + EntryUtil.ENTRY_SEPARATOR - + "myid1"; - - static protected String fileEntry = - EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR - + "myfile" + EntryUtil.ENTRY_SEPARATOR - + "myid2"; - - static protected String linkEntry = - EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR - + "mylink" + EntryUtil.ENTRY_SEPARATOR - + "myid3"; - - static protected String metaBlock = - "12345" + EntryUtil.BLOCKIDS_SEPARATOR - + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR - + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR - + "myid3"; - - @Test - public void testIsDirectory() { - assertTrue(EntryUtil.isDirectory(directoryEntry)); - assertFalse(EntryUtil.isDirectory(fileEntry)); - } - - @Test - public void testFind() { - - String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR - + fileEntry + EntryUtil.ENTRIES_SEPARATOR - + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - - // seek and return entry - String findResult; - - findResult = EntryUtil.findEntryInDirectory(content, "mydir"); - assertEquals(directoryEntry, findResult); - findResult = EntryUtil.findEntryInDirectory(content, "myfile"); - assertEquals(fileEntry, findResult); - findResult = EntryUtil.findEntryInDirectory(content, "mylink"); - assertEquals(linkEntry, findResult); - - - // return null if file not found - findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); - assertEquals(null, findResult); - } - - @Test - public void testGetId() { - assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry)); - assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry)); - assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry)); - } - - @Test - public void testGetName() { - assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry)); - assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry)); - assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry)); - - } - - @Test - public void testGetType() { - assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry)); - assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry)); - assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry)); - } - - @Test - public void testResolveLink() { - String path; - - path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); - assertEquals("/anotherdir", path); - path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); - assertEquals("/dir/subdir/anotherdir", path); - path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); - assertEquals("/dir/anotherdir", path); - path = EntryUtil.resolveLink("/dir/", "subdir/file"); - assertEquals("/dir/subdir/file", path); - } - - @Test - public void testBytesToArray() { - String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$"; - String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); - assertEquals(s, copy); - } - - @Test - public void testGetTotalSizeFromMetaBlock() { - assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); - } - - @Test - public void testGetBlockIdsFromMetaBlock() { - String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; - String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); - assertArrayEquals(expectedBlocksIds, actualBlocksIds); - } - - @Test - public void testRemoveEntryFromEntries() { - - String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR - + fileEntry + EntryUtil.ENTRIES_SEPARATOR - + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - - String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR - + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - - String actual = EntryUtil.removeEntryFromEntries(content, "myfile"); - - assertEquals(expected, actual); - - } -} Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,27 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertFalse; - -import java.util.Arrays; - -import org.junit.Test; -import org.nuiton.disworkfs.storage.InMemoryMap; - -public class InMemoryMapTest { - - /** - * this test show that the InMemoryMap put implies a copy of the data. - * After the put, the original value is modified. A final check shows - * there is no side effect - */ - @Test - public void testPut() { - InMemoryMap map = new InMemoryMap(); - byte[] expected = {0x1, 0x2}; - map.put("key", expected); - expected[0] = 0xf; - byte[] actual = map.get("key"); - assertFalse(Arrays.equals(expected, actual)); - } - -} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,89 @@ +package org.nuiton.disworkfs; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + +/** + * This class provides utilities to write tests easily. + * + * You can get multiples diswork configs ready to use. All the instances uses + * a new port and the local IP. + * <pre> + * c = newDisworkConfig(); // create a config for a bootstrap node + * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that + * // will bootstrap by joining the + * // first node + * c3 = newDisworkConfig(c.getUsedPort()) + * </pre> + * + */ +public class Util { + + protected static Integer port = 9000; + + /** + * returns a new port, returned value change at each call. + * @return + */ + public static Integer getPort() { + port += 1; + return port; + } + + /** + * returns the IP on the local machine. Trying to get an public IP or a LAN + * IP or the loopback IP if there is no other interface + * @return + * @throws UnknownHostException + */ + public static InetAddress getIp() throws UnknownHostException { + InetAddress result = InetAddress.getLocalHost(); + if (result.isLoopbackAddress()) { + try { + Socket temp = new Socket("microsoft.com", 80); + result = temp.getLocalAddress(); + temp.close(); + } catch (IOException e) { + // TODO 20100602 bleny do something ? + } + } + return result; + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a single-node instance of DisworkFS + * @return + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newDisworkConfig() + throws UnknownHostException { + return newDisworkConfig(null); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a multiple-node instance of DisworkFS on a same machine. + * @param bootstrapPort the port on the same machine where another node can + * be found to bootstrap + * @return a complete config + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = Util.getPort().toString(); + String ip = getIp().getHostAddress(); + result.setOption("diswork.fs.use_in_memory_map", "false"); + result.setOption("diswork.fs.use_port", port); + result.setOption("diswork.fs.bootstrap.ip", ip); + if (bootstrapPort == null) { + result.setOption("diswork.fs.bootstrap.port", port); + } else { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + } + return result; + } +} \ No newline at end of file Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java (from rev 52, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,159 @@ +/* *##% + * Copyright (c) 2010 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.nuiton.disworkfs.storage; + + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.nuiton.disworkfs.storage.EntryUtil; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class EntryUtilTest { + + static protected String directoryEntry = + EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "mydir" + EntryUtil.ENTRY_SEPARATOR + + "myid1"; + + static protected String fileEntry = + EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR + + "myfile" + EntryUtil.ENTRY_SEPARATOR + + "myid2"; + + static protected String linkEntry = + EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR + + "mylink" + EntryUtil.ENTRY_SEPARATOR + + "myid3"; + + static protected String metaBlock = + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid3"; + + @Test + public void testIsDirectory() { + assertTrue(EntryUtil.isDirectory(directoryEntry)); + assertFalse(EntryUtil.isDirectory(fileEntry)); + } + + @Test + public void testFind() { + + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + // seek and return entry + String findResult; + + findResult = EntryUtil.findEntryInDirectory(content, "mydir"); + assertEquals(directoryEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "mylink"); + assertEquals(linkEntry, findResult); + + + // return null if file not found + findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, findResult); + } + + @Test + public void testGetId() { + assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry)); + assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry)); + assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry)); + } + + @Test + public void testGetName() { + assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry)); + assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry)); + assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry)); + + } + + @Test + public void testGetType() { + assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry)); + assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry)); + assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry)); + } + + @Test + public void testResolveLink() { + String path; + + path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); + assertEquals("/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); + assertEquals("/dir/subdir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); + assertEquals("/dir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/", "subdir/file"); + assertEquals("/dir/subdir/file", path); + } + + @Test + public void testBytesToArray() { + String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$"; + String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); + assertEquals(s, copy); + } + + @Test + public void testGetTotalSizeFromMetaBlock() { + assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); + } + + @Test + public void testGetBlockIdsFromMetaBlock() { + String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; + String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); + assertArrayEquals(expectedBlocksIds, actualBlocksIds); + } + + @Test + public void testRemoveEntryFromEntries() { + + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + String actual = EntryUtil.removeEntryFromEntries(content, "myfile"); + + assertEquals(expected, actual); + + } +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java (from rev 52, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,27 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertFalse; + +import java.util.Arrays; + +import org.junit.Test; +import org.nuiton.disworkfs.storage.InMemoryDisworkMap; + +public class InMemoryMapTest { + + /** + * this test show that the InMemoryMap put implies a copy of the data. + * After the put, the original value is modified. A final check shows + * there is no side effect + */ + @Test + public void testPut() { + InMemoryDisworkMap map = new InMemoryDisworkMap(); + byte[] expected = {0x1, 0x2}; + map.put("key", expected); + expected[0] = 0xf; + byte[] actual = map.get("key"); + assertFalse(Arrays.equals(expected, actual)); + } + +} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,98 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.disworkfs.Util; + +public class PastryDisworkMapTest { + + protected static PastryDisworkMap map1; + protected static PastryDisworkMap map2; + + protected static byte[] bytes; + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + /* + if (map1 != null) + map1.close(); + if (map2 != null) + map2.close(); + */ + } + + @Test + public void test() throws Exception { + DisworkFileSystemConfig config = Util.newDisworkConfig(); + + map1 = new PastryDisworkMap(config); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + bytes = newBytes; + + map1.put("test", newBytes); + + byte[] getResult = map1.get("test"); + assertArrayEquals(newBytes, getResult); + } + + @Test + public void test2nodes() throws Exception { + DisworkFileSystemConfig config1 = Util.newDisworkConfig(); + map1 = new PastryDisworkMap(config1); + + DisworkFileSystemConfig config2 = Util.newDisworkConfig(config1.getUsedPort()); + map2 = new PastryDisworkMap(config2); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + bytes = newBytes; + + map1.put("test", newBytes); + + assertTrue(map2.containsKey("test")); + assertFalse(map2.containsKey("unknownkey")); + + byte[] getResult = map2.get("test"); + + assertArrayEquals(newBytes, getResult); + + byte[] removeResult = map2.remove("test"); + assertArrayEquals(newBytes, removeResult); + + map1.put("test", newBytes); + map2.put("test", newBytes); + + map2.remove("unknow"); + + } + + @Test + public void testClose() throws Exception { + /* + DisworkFileSystemConfig config1 = new DisworkFileSystemConfig(); + map1 = new PastryDisworkMap(config1); + */ + /* + try { + map2 = new PastryDisworkMap(config1); + } catch (IllegalStateException e) { + assertEquals("Cannot bind to /192.168.99.119:9001", e.getMessage()); + } + */ + + // map1.close(); + // map2 = new PastryDisworkMap(config1); + } + +}

1 0

r54 - trunk
by tchemit＠users.nuiton.org June 2, 2010

June 2, 2010

Author: tchemit Date: 2010-06-02 12:27:22 +0200 (Wed, 02 Jun 2010) New Revision: 54 Url: http://nuiton.org/repositories/revision/diswork/54 Log: Utilisation de mavenpom4redmine 2.1.5 Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-05-26 08:44:36 UTC (rev 53) +++ trunk/pom.xml 2010-06-02 10:27:22 UTC (rev 54) @@ -10,7 +10,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>2.1.4</version> + <version>2.1.5</version> </parent> <artifactId>diswork</artifactId>

1 0