Author: bleny Date: 2010-06-02 14:56:07 +0200 (Wed, 02 Jun 2010) New Revision: 55 Url: http://nuiton.org/repositories/revision/diswork/55 Log: implementation of DisworkMap with FreePastry ; more documentation Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/resources/freepastry.params trunk/diswork-fs/src/main/resources/tester.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/pom.xml 2010-06-02 12:56:07 UTC (rev 55) @@ -1,4 +1,5 @@ -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.nuiton</groupId> @@ -11,20 +12,20 @@ <packaging>jar</packaging> <name>disworkfs</name> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>2.0.2</version> - <configuration> - <source>1.6</source> - <target>1.6</target> - </configuration> - </plugin> - </plugins> - </build> - <dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.0.2</version> + <configuration> + <source>1.6</source> + <target>1.6</target> + </configuration> + </plugin> + </plugins> + </build> + <dependencies> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> @@ -41,12 +42,15 @@ <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> -<!-- <dependency> + <groupId>org.freepastry</groupId> + <artifactId>pastry</artifactId> + </dependency> + <dependency> <groupId>fr.inria.peerunit</groupId> <artifactId>PeerUnit</artifactId> </dependency> ---> + <!-- test --> <dependency> <groupId>junit</groupId> Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,221 @@ +package org.nuiton.disworkfs; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Random; + +import javax.swing.plaf.SliderUI; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.runner.Runner; + +public class Demo { + + private static final Log log = LogFactory.getLog(Demo.class); + + protected static DisworkFileSystem fileSystem; + + protected static class Producer implements Runnable { + + protected String waitingForJob = null; + protected Integer expectedResult = null; + + @Override + public void run() { + while (true) { + + try { + if (fileSystem.exists("/todo")) { + + if (waitingForJob == null) { + + // taking a random job + Random random = new Random(); + + waitingForJob = "/todo/job-" + random.nextInt(); + Integer randomInteger = random.nextInt(); + expectedResult = randomInteger + 1; + String job = randomInteger.toString(); + + InputStream source = IOUtils.toInputStream(job); + + + log.info("writing job " + waitingForJob + " (" + randomInteger + ")"); + + fileSystem.write(waitingForJob, source); + + source.close(); + + List<String> todos = fileSystem.readDirectory("/todo"); + + System.out.println("number of jobs : " + todos.size()); + + } else { + String resultPath = waitingForJob + ".result"; + if (fileSystem.exists(resultPath)) { + log.info("results has been published"); + + InputStream resultStream = fileSystem.read(resultPath); + + String resultString = IOUtils.toString(resultStream); + + Integer result = Integer.parseInt(resultString); + + log.info("result is " + result + "(expected : " + expectedResult + ")"); + + fileSystem.delete(resultPath); + fileSystem.delete(waitingForJob); + + waitingForJob = null; + expectedResult = null; + + } + + } + + + } else { + fileSystem.createDirectory("/todo"); + } + + } catch (IOException e) { + log.error(e); + } + + try { + Thread.sleep(15 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + + } + } + } + + protected static class Consumer implements Runnable { + + @Override + public void run() { + while (true) { + + try { + if (fileSystem.exists("/todo")) { + + List<String> todos = fileSystem.readDirectory("/todo"); + + if (todos.isEmpty()) { + log.info("nothing to do"); + } else { + // taking a random job + Random random = new Random(); + String todo = todos.get(random.nextInt(todos.size())); + + if (!todo.endsWith(".result")) { + String jobPath = "/todo/" + todo; + log.info("reading the job " + jobPath); + + InputStream in = fileSystem.read(jobPath); + + String operation = IOUtils.toString(in); + + // in.close(); + + log.info("operation to do " + operation); + + Integer i = Integer.parseInt(operation); + + i += 1; + + String result = i.toString(); + + log.info("result is " + result); + + InputStream source = IOUtils.toInputStream(result); + + fileSystem.write(jobPath + ".result", source); + + source.close(); + + } + } + + } else { + fileSystem.createDirectory("/todo"); + } + + } catch (IOException e) { + log.error(e); + } + + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + + } + } + } + + public static void printUsage() { + System.out.println( + "Usage :\n" + + "org.nuiton.disworkfs.Demo producer|consumer usePort [bootStrapIp bootStrapPort]" + ); + } + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + if (args.length == 2) { + + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + + config.setOption("diswork.fs.use_port", args[1]); + + fileSystem = new DisworkFileSystem(config); + + if ("producer".equals(args[0])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[0])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + + } else if (args.length == 4) { + + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + + config.setOption("diswork.fs.use_port", args[1]); + config.setOption("diswork.fs.bootstrap.ip", args[2]); + config.setOption("diswork.fs.bootstrap.port", args[3]); + + fileSystem = new DisworkFileSystem(config); + + if ("producer".equals(args[2])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[2])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + + } else { + printUsage(); + } + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-02 12:56:07 UTC (rev 55) @@ -11,13 +11,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; -import org.nuiton.disworkfs.storage.InMemoryMap; import org.nuiton.disworkfs.storage.Storage; /** Main class of Diswork File System, provide methods for all operations * You can use: * <ul> - * <li>{@link #createDirectory(String)} and {@link DisworkFileSystem#readDirectory(String)} + * <li>{@link #createDirectory(String)} and {@link #readDirectory(String)} * to create and browse directories</li> * <li>{@link #write(String, InputStream)} and {@link #read(String)} * to write a file and read it</li> @@ -44,13 +43,12 @@ /** constructor allowing to provide another configuration to use * - * @param disworkConfig the configuration + * @param config the configuration * @throws IOException */ - public DisworkFileSystem(DisworkFileSystemConfig disworkConfig) + public DisworkFileSystem(DisworkFileSystemConfig config) throws IOException { - storage = new Storage(disworkConfig, new InMemoryMap()); - storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + storage = new Storage(config); } /** tests the existence of a file/dir/link at a given path @@ -98,6 +96,9 @@ String id = EntryUtil.getIdFromEntry(entry); result = storage.getFile(id); } + + log.info("read " + path + " returns " + result.available() + " bytes"); + return result; } @@ -116,6 +117,7 @@ } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); + log.info("writing " + source.available() + " bytes at " + path); write(parent, name, source); } @@ -186,6 +188,7 @@ // store file before meta info storage.putDirectory(newDirectoryId, EntryUtil.newEmptyDirectoryContent()); + // update meta info directory storage.putDirectory(parentId, newContent); } else { @@ -329,18 +332,25 @@ entry = walk(target); } if (EntryUtil.isDirectory(entry)) { + result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); - String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR); - result = new ArrayList<String>(); - for (String elementEntry : entries) { - result.add(EntryUtil.getNameFromEntry(elementEntry)); + if (EntryUtil.newEmptyDirectoryContent().equals(content)) { + // directory is empty, add nothing + } else { + String[] entries = content.split( + EntryUtil.ENTRIES_SEPARATOR); + for (String elementEntry : entries) { + result.add(EntryUtil.getNameFromEntry(elementEntry)); + } } } else { throw new IOException(path + " is not a directory"); } } + log.info("readDirectory " + path + " returns " + result.size() + " results"); + return result; } @@ -418,7 +428,7 @@ log.info("in intermediate dir " + current + ", looking for " + p); // mise a jour de current - if (current.equals("/")) current = ""; + if (current.equals("/")) current = ""; // avoid "//path" next line current += "/" + p; String entry = EntryUtil.findEntryInDirectory(content, p); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-02 12:56:07 UTC (rev 55) @@ -15,12 +15,32 @@ * </dl> */ public class DisworkFileSystemConfig extends ApplicationConfig { - + public DisworkFileSystemConfig() { - setDefaultOption("blocks_size", "10485760"); // 10 MiB + setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB + setDefaultOption("diswork.fs.use_in_memory_map", "false"); + setDefaultOption("diswork.fs.use_port", "9001"); + setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); + setDefaultOption("diswork.fs.bootstrap.port", "9001"); } public int getBlockSize() { - return getOptionAsInt("blocks_size"); + return getOptionAsInt("diswork.fs.blocks_size"); } + + public boolean useInMemoryMap() { + return getOptionAsBoolean("diswork.fs.use_in_memory_map"); + } + + public Integer getUsedPort() { + return getOptionAsInt("diswork.fs.use_port"); + } + + public String getBootstrapIp() { + return getOption("diswork.fs.bootstrap.ip"); + } + + public Integer getBootstrapPort() { + return getOptionAsInt("diswork.fs.bootstrap.port"); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,10 +1,11 @@ /** * DisworkFS is a distributed file system. You can use it by instantiating - * {@link DisworkFileSystem} and use it to store directories and their content. + * {@link org.nuiton.disworkfs.DisworkFileSystem} and use it to store + * directories and their content. * * You can change the default DisworkFileSystem behavior by provide a - * {@link DisworkFileSystemConfig} instance at construction. If you don't - * provide one, a instance will be created + * {@link org.nuiton.disworkfs.DisworkFileSystemConfig} instance at + * construction. If you don't provide one, a instance will be created */ package org.nuiton.disworkfs; \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,93 @@ +package org.nuiton.disworkfs.peerunit; + +import static fr.inria.peerunit.test.assertion.Assert.assertTrue; +import static fr.inria.peerunit.test.assertion.Assert.assertEquals; +import static fr.inria.peerunit.test.assertion.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Arrays; + +import org.apache.commons.io.IOUtils; +import org.nuiton.disworkfs.DisworkFileSystem; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +import fr.inria.peerunit.TestCaseImpl; +import fr.inria.peerunit.parser.TestStep; + +public class DisworkFileSystemTest extends TestCaseImpl { + + protected Integer port = 9001; + + protected DisworkFileSystem fileSystem; + + protected byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; + + @TestStep(range="0",timeout=1000000, order = 0) + public void testConnect() throws Exception { + Integer myPort = port + getId(); + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + config.setOption("diswork.fs.use_port", myPort.toString()); + fileSystem = new DisworkFileSystem(config); + } + + @TestStep(range="1",timeout=1000000, order = 1) + public void testConnect1() throws Exception { + Integer myPort = port + getId(); + DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + config.setOption("diswork.fs.use_port", myPort.toString()); + fileSystem = new DisworkFileSystem(config); + } + /* + @TestStep(range="0",timeout=1000000, order = 2) + public void testWrite() throws Exception { + InputStream source = new ByteArrayInputStream(bytes); + fileSystem.write("/myfile", source); + source.close(); + } + + @TestStep(range="1",timeout=1000000, order = 3) + public void testRead() throws Exception { + try { + InputStream source = fileSystem.read("/myfile"); + byte[] readResult = IOUtils.toByteArray(source); + source.close(); + boolean arraysContentEquals = Arrays.equals(bytes, readResult); + assertTrue(arraysContentEquals); + } catch (FileNotFoundException e) { + fail(); + } + } + */ + @TestStep(range="1",timeout=1000000, order = 4) + public void testCreateDir() throws Exception { + fileSystem.createDirectory("/mydir"); + fileSystem.createDirectory("/myseconddir"); + } + + @TestStep(range="0-1",timeout=1000000, order = 5) + public void fake() throws Exception { + + } + + /* + @TestStep(range="0-1",timeout=1000000, order = 5) + public void testCreateDir2() throws Exception { + assertTrue(fileSystem.exists("/mydir")); + assertTrue(fileSystem.exists("/myseconddir")); + } + */ + /* + @TestStep(range="1",timeout=1000000, order = 6) + public void testCreateSubDir() throws Exception { + fileSystem.createDirectory("/mydir/mysubdir"); + } + + @TestStep(range="1",timeout=1000000, order = 7) + public void testReadDir() throws Exception { + assertTrue(fileSystem.exists("/mydir/mysubdir")); + assertEquals(1, fileSystem.readDirectory("/mydir").size()); + } + */ +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,6 @@ +package org.nuiton.disworkfs.storage; + +import java.io.Closeable; +import java.util.Map; + +public interface DisworkMap extends Map<String, byte[]>, Closeable {} Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java (from rev 52, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,32 @@ +package org.nuiton.disworkfs.storage; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** An in-memory implementation of a Map + * This class is a map where put make a copy of the value. It is used for + * stubbing purpose. It emulates the behavior of a DHT and thus can be used + * by {@link Storage} to store data and blocks. + */ +public class InMemoryDisworkMap extends HashMap<String, byte[]> + implements DisworkMap { + + private final Log log = LogFactory.getLog(InMemoryDisworkMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } + + @Override + public void close() throws IOException { + // nothing to do + } +} \ No newline at end of file Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,25 +0,0 @@ -package org.nuiton.disworkfs.storage; - -import java.util.Arrays; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** An in-memory implementation of a Map - * This class is a map where put make a copy of the value. It is used for - * stubbing purpose. It emulates the behavior of a DHT and thus can be used - * by {@link Storage} to store data and blocks. - */ -public class InMemoryMap extends HashMap<String, byte[]> { - - private final Log log = LogFactory.getLog(InMemoryMap.class); - - @Override - public byte[] put(String key, byte[] value) { - log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); - byte[] valueCopy = value.clone(); - log.info("putting " + valueCopy.length + " bytes"); - return super.put(key, valueCopy); - } -} \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,363 @@ +package org.nuiton.disworkfs.storage; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +import rice.Continuation; +import rice.environment.Environment; +import rice.p2p.commonapi.Id; +import rice.p2p.past.ContentHashPastContent; +import rice.p2p.past.PastContent; +import rice.p2p.past.PastImpl; +import rice.pastry.NodeIdFactory; +import rice.pastry.PastryNode; +import rice.pastry.PastryNodeFactory; +import rice.pastry.commonapi.PastryIdFactory; +import rice.pastry.socket.SocketPastryNodeFactory; +import rice.pastry.standard.RandomNodeIdFactory; +import rice.persistence.EmptyCache; +import rice.persistence.MemoryStorage; +import rice.persistence.StorageManagerImpl; + +/** + * An implementation of {@link DisworkMap} using FreePastry. + * + * @see <a href="https://trac.freepastry.org/">FreePastry site</a> + * + */ +public class PastryDisworkMap implements DisworkMap { + + private final Log log = LogFactory.getLog(PastryDisworkMap.class); + + protected static class ByteArrayPastContent extends ContentHashPastContent { + + private static final long serialVersionUID = 7584834640788361309L; + + byte[] content; + + public ByteArrayPastContent(Id id, byte[] content) { + super(id); + this.content = content; + } + + public String toString() { + return "ByteArrayPastContent (" + content.length + " bytes)"; + } + } + + protected abstract class MyContinuation<R, E extends Exception> implements Continuation<R, Exception> { + + protected boolean finished = false; + + public boolean isFinished() { + return finished; + } + + public void waitUntilFinised() { + while(!finished) { + try { + Thread.sleep(1500); + log.info("waiting for response"); + } catch (InterruptedException e) { + // TODO 20100527 bleny stub + log.info("exception catch", e); + e.printStackTrace(); + } + } + } + + @Override + public void receiveException(Exception exception) { + log.error("exception received : " + exception); + finished = true; + } + + @Override + public void receiveResult(R result) { + finished = true; + } + + } + + protected PastImpl past; + + protected PastryIdFactory pastryIdFactory; + + public PastryDisworkMap(DisworkFileSystemConfig config) throws IOException { + + Environment env = new Environment(); + + // disable the UPnP setting (in case you are testing this on a NATted LAN) + // env.getParameters().setString("nat_search_policy", "never"); + + // the port to use locally + int bindport = config.getUsedPort(); + + InetAddress bootaddr = InetAddress.getByName(config.getBootstrapIp()); + int bootport = config.getBootstrapPort(); + + InetSocketAddress bootaddress = new InetSocketAddress(bootaddr, bootport); + + log.info("boot address : " + bootaddress); + + NodeIdFactory nidFactory = new RandomNodeIdFactory(env); + + // construct the PastryNodeFactory, this is how we use rice.pastry.socket + PastryNodeFactory factory = null; + + // FIXME 20100602 bleny this is a work-around to deal with time out on + // connect exceptions that occurs "sometimes" + int numberOfTry = 0; + while(factory == null && numberOfTry <= 10) { + try { + factory = new SocketPastryNodeFactory(nidFactory, bindport, env); + } catch (java.net.ConnectException e) { + numberOfTry += 1; + } + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + log.info("exception catch", e); + e.printStackTrace(); + } + } + if (factory == null) { + throw new IOException("unable to connect"); + } + + // construct a node, but this does not cause it to boot + PastryNode node = factory.newNode(); + + // in later tutorials, we will register applications before calling boot + node.boot(bootaddress); + + // the node may require sending several messages to fully boot into the ring + synchronized(node) { + while(!node.isReady() && !node.joinFailed()) { + // delay so we don't busy-wait + try { + node.wait(500); + } catch (InterruptedException e) { + // TODO 20100528 bleny Auto-generated catch block + log.info("exception catch", e); + e.printStackTrace(); + } + + // abort if can't join + if (node.joinFailed()) { + throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason()); + } + } + } + + log.info("finished creating new node " + node); + + pastryIdFactory = new PastryIdFactory(env); + + rice.persistence.Storage storage = new MemoryStorage(pastryIdFactory); + past = new PastImpl( + node, + new StorageManagerImpl( + pastryIdFactory, + storage, + // Using a cache do not permit to remove while putting + /* + new LRUCache( + new MemoryStorage(pastryIdFactory), + 512 * 1024, + node.getEnvironment() + ) + */ + new EmptyCache(pastryIdFactory) + ), + 0, // replication factor + "" + ); + + + } + + protected class ContainsKeyContinuation extends MyContinuation<Boolean, Exception> { + + public Boolean result = null; + + @Override + public void receiveResult(Boolean result) { + log.info("contains key result received : " + result); + this.result = result; + finished = true; + } + } + + @Override + public boolean containsKey(Object key) { + Id id = pastryIdFactory.buildId((String) key); + ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation(); + past.existsInOverlay(id, containsKeyContinuation); + containsKeyContinuation.waitUntilFinised(); + boolean result = containsKeyContinuation.result; + log.info("containsKey " + key + " (id = " + id + ") returns " + result); + return result; + } + + protected class GetContinuation extends MyContinuation<PastContent, Exception> { + + public byte[] result = null; + + @Override + public void receiveResult(PastContent result) { + log.info("get result received " + result); + if (result != null) { + this.result = ((ByteArrayPastContent) result).content; + } + finished = true; + } + } + + protected byte[] atomicGet(Object key) { + Id id = pastryIdFactory.buildId((String) key); + GetContinuation getContinuation = new GetContinuation(); + past.lookup(id, getContinuation); + getContinuation.waitUntilFinised(); + return getContinuation.result; + } + + @Override + public byte[] get(Object key) { + byte[] result = atomicGet(key); + if (result == null) { + log.info("get " + key + " returns null"); + } else { + log.info("get " + key + " returns " + result.length + " bytes"); + } + return result; + } + + protected class PutContinuation extends MyContinuation<Boolean[], Exception> { + @Override + public void receiveResult(Boolean[] result) { + log.info("insert result received : " + Arrays.toString(result)); + finished = true; + } + } + + protected void atomicPut(String key, byte[] value) { + Id id = pastryIdFactory.buildId(key); + PastContent pastContent = new ByteArrayPastContent(id, value); + PutContinuation putContinuation = new PutContinuation(); + past.insert(pastContent, putContinuation); + putContinuation.waitUntilFinised(); + } + + @Override + public byte[] put(String key, byte[] value) { + byte[] previousValue = atomicGet(key); + if (previousValue != null) { + atomicRemove(key); + } + atomicPut(key, value); + log.info("put " + value.length + " bytes at " + key ); + return previousValue; + } + + protected class RemoveContinuation extends MyContinuation<Boolean, Exception> { + + public Boolean result = null; + + @Override + public void receiveResult(Boolean result) { + log.info("remove result received : " + result); + this.result = result; + finished = true; + } + } + + protected void atomicRemove(Object key) { + Id id = pastryIdFactory.buildId((String) key); + RemoveContinuation removeContinuation = new RemoveContinuation(); + past.remove(id, removeContinuation); + removeContinuation.waitUntilFinised(); + log.info("atomic remove " + key + " has returned " + removeContinuation.result); + } + + @Override + public byte[] remove(Object key) { + byte[] previousValue = atomicGet(key); + if (previousValue == null) { + log.info("remove " + key + " returns null"); + } else { + atomicRemove(key); + log.info("remove " + key + " returns " + previousValue.length + " bytes"); + } + return previousValue; + } + + @Override + public void close() throws IOException { + // TODO 20100528 bleny should close Pastry and close all sockets + // throw new UnsupportedOperationException("not yet implemented"); + /* + ServerSocket s = new ServerSocket(9001); + s.getChannel().close(); + s.close(); + */ + } + + @Deprecated + @Override + public void putAll(Map<? extends String, ? extends byte[]> arg0) { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public int size() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Collection<byte[]> values() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public boolean isEmpty() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Set<String> keySet() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public boolean containsValue(Object arg0) { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Set<java.util.Map.Entry<String, byte[]>> entrySet() { + throw new UnsupportedOperationException(); + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-02 12:56:07 UTC (rev 55) @@ -15,7 +15,11 @@ import org.nuiton.disworkfs.DisworkFileSystemConfig; /** + * This class is the middle layer between the File System operations and + * the Map<String, byte[]>. It offers put and get on directories, files + * and links and deal with splitting consideration and concurrency * + * It needs a place to store data called a {@link DisworkMap}. */ public class Storage implements Closeable { @@ -104,13 +108,23 @@ } - protected Map<String, byte[]> map; + protected DisworkMap map; protected DisworkFileSystemConfig disworkConfig; - public Storage(DisworkFileSystemConfig disworkConfig, Map<String, byte[]> map) { - this.map = map; + public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { this.disworkConfig = disworkConfig; + if (disworkConfig.useInMemoryMap()) { + log.info("using in-memory map"); + this.map = new InMemoryDisworkMap(); + putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + } else { + log.info("using Pastry map"); + this.map = new PastryDisworkMap(disworkConfig); + if (! map.containsKey("/")) { + putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + } + } } /** @@ -189,30 +203,36 @@ // trying to acquire lock Long currentDate = System.currentTimeMillis(); - byte[] lock = map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString())); + log.info("trying to acquire a lock on " + key); + byte[] lock = map.put(lockKey, + EntryUtil.stringToBytes(currentDate.toString())); if (lock != null) { // file is locked, check date Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock)); - - if (currentDate - currentLock > LOCK_VALID_TIME) { - // this lock is out-dated, let's erase it + Long lockAge = currentDate - currentLock; + if (lockAge > LOCK_VALID_TIME) { + log.info("lock is out-dated (" + lockAge + " ms old)"); + // this lock is out-dated, let's erase all data String obsoleteMetaBlock = EntryUtil.bytesToString(map.get(newDataKey)); if (obsoleteMetaBlock != null) { String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); + log.info("removing " + obsoleteBlocksIds.length + " old blocks"); for (String obsoleteBlockId : obsoleteBlocksIds) { map.remove(obsoleteBlockId); } } } else { - // file is currently written, stopping operation + log.info(key + " is currently written, stopping operation"); map.put(lockKey, lock); throw new ConcurrentModificationException("can't write due to concurrency"); } } + log.info("lock on " + key + " acquired"); + // here, we know we can write or an exception // would have been thrown earlier String blocksIds = ""; @@ -258,6 +278,8 @@ } map.put(key, map.get(newDataKey)); + + log.info("release lock " + lockKey); map.remove(lockKey); } @@ -283,6 +305,7 @@ @Override public void close() throws IOException { clean(); + map.close(); } } \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,7 +1,8 @@ /** * <p> * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a - * way to persistently store data. This is done by the {@link Storage} class + * way to persistently store data. This is done by the + * {@link org.nuiton.disworkfs.storage.Storage} class * which permit to store different type of data. * </p> * <p> @@ -82,12 +83,14 @@ * entry (<strong>D</strong>irectory, <strong>F</strong>ile or * <strong>L</strong>ink), the name of the element, and an ID to be used as * a key on the map to get the actual content. Those three informations are - * separated by ":" which is {@link EntryUtil#ENTRY_SEPARATOR}. + * separated by ":" which is + * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRY_SEPARATOR}. * </p> * * <p> * A directory way have multiple <em>entries</em>. Entries of a directory are - * separated by "\n" which is {@link EntryUtil#ENTRIES_SEPARATOR}. + * separated by "\n" which is + * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRIES_SEPARATOR}. * </p> * <p> * The above description shows the main principle used to store a tree @@ -171,7 +174,7 @@ * A metablock is composed of the total size of the file followed by * an ordered lists of IDs of the different blocks composing the file. * Those informations are separated by ";" (see - * {@link EntryUtil#BLOCKIDS_SEPARATOR}). + * {@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). * </p> * <p> * In the above example: @@ -194,9 +197,10 @@ * <p> * When reading and writing in storage, split is done transparently. When * reading, a Stream is returned: it loads data blocks after blocks - * when needed (see {@link Storage.SplitBlocksInputStream}). When writing, - * data are split in blocks of a maximum configurable size - * (see {@link org.nuiton.disworkfs.DisworkConfig}). + * when needed (see + * {@link org.nuiton.disworkfs.storage.Storage.SplitBlocksInputStream}). + * When writing, data are split in blocks of a maximum configurable size + * (see {@link org.nuiton.disworkfs.DisworkFileSystemConfig}). * </p> */ Added: trunk/diswork-fs/src/main/resources/freepastry.params =================================================================== --- trunk/diswork-fs/src/main/resources/freepastry.params (rev 0) +++ trunk/diswork-fs/src/main/resources/freepastry.params 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,404 @@ +# this file holds the default values for pastry and it's applications +# you do not need to modify the default.params file to override these values +# instead you can use your own params file to set values to override the +# defaults. You can specify this file by constructing your +# rice.environment.Environment() with the filename you wish to use +# typically, you will want to be able to pass this file name from the command +# line + +# max number of handles stored per routing table entry +pastry_rtMax = 1 +pastry_rtBaseBitLength = 4 + +# leafset size +pastry_lSetSize = 24 + +# maintenance frequencies +pastry_leafSetMaintFreq = 60 +pastry_routeSetMaintFreq = 900 + +# drop the message if pastry is not ready +pastry_messageDispatch_bufferIfNotReady = false + +# number of messages to buffer while an app hasn't yet been registered +pastry_messageDispatch_bufferSize = 32 + +# FP 2.1 uses the new transport layer +transport_wire_datagram_receive_buffer_size = 131072 +transport_wire_datagram_send_buffer_size = 65536 +transport_epoch_max_num_addresses = 2 +transport_sr_max_num_hops = 5 + +# proximity neighbor selection +transport_use_pns = true + +# number of rows in the routing table to consider during PNS +# valid values are ALL, or a number +pns_num_rows_to_use = 10 + +# commonapi testing parameters + +# direct or socket +commonapi_testing_exit_on_failure = true +commonapi_testing_protocol = direct +commonapi_testing_startPort = 5009 +commonapi_testing_num_nodes = 10 +# set this to specify the bootstrap node +#commonapi_testing_bootstrap = localhost:5009 + +# random number generator's seed, "CLOCK" uses the current clock time +random_seed = CLOCK + +# sphere, euclidean or gt-itm +direct_simulator_topology = sphere +# -1 starts the simulation with the current time +direct_simulator_start_time = -1 +#pastry_direct_use_own_random = true +#pastry_periodic_leafset_protocol_use_own_random = true +pastry_direct_gtitm_matrix_file=GNPINPUT +# the number of stubs in your network +pastry_direct_gtitm_max_overlay_size=1000 +# the number of virtual nodes at each stub: this allows you to simulate multiple "LANs" and allows cheeper scaling +pastry_direct_gtitm_nodes_per_stub=1 +# the factor to multiply your file by to reach millis. Set this to 0.001 if your file is in microseconds. Set this to 1000 if your file is in seconds. +pastry_direct_gtitm_delay_factor=1.0 +#millis of the maximum network delay for the generated network topologies +pastry_direct_max_diameter=200 +pastry_direct_min_delay=2 +#setting this to false will use the old protocols which are about 200 times as fast, but may cause routing inconsistency in a real network. Probably won't in a simulator because it will never be incorrect about liveness +pastry_direct_guarantee_consistency=true + +# rice.pastry.socket parameters +# tells the factory you intend to use multiple nodes +# this causes the logger to prepend all entries with the nodeid +pastry_factory_multipleNodes = true +pastry_factory_selectorPerNode = false +pastry_factory_processorPerNode = false +# number of bootstap nodehandles to fetch in parallel +pastry_factory_bootsInParallel = 1 + +# the maximum size of a message +pastry_socket_reader_selector_deserialization_max_size = 1000000 +# the maximum number of outgoing messages to queue when a socket is slower than the number of messages you are queuing +pastry_socket_writer_max_queue_length = 30 +pastry_socket_writer_max_msg_size = 20480 +pastry_socket_repeater_buffer_size = 65536 +pastry_socket_pingmanager_smallPings=true +pastry_socket_pingmanager_datagram_receive_buffer_size = 131072 +pastry_socket_pingmanager_datagram_send_buffer_size = 65536 +# the time before it will retry a route that was already found dead +pastry_socket_srm_check_dead_throttle = 300000 +pastry_socket_srm_proximity_timeout = 3600000 +pastry_socket_srm_ping_throttle = 30000 +pastry_socket_srm_default_rto = 3000 +pastry_socket_srm_rto_ubound = 10000 +pastry_socket_srm_rto_lbound = 50 +pastry_socket_srm_gain_h = 0.25 +pastry_socket_srm_gain_g = 0.125 +pastry_socket_scm_max_open_sockets = 300 +pastry_socket_scm_max_open_source_routes = 30 +# the maximum number of source routes to attempt, setting this to 0 will +# effectively eliminate source route attempts +# setting higher than the leafset does no good, it will be bounded by the leafset +# a larger number tries more source routes, which could give you a more accurate +# determination, however, is more likely to lead to congestion collapse +pastry_socket_srm_num_source_route_attempts = 8 +pastry_socket_scm_socket_buffer_size = 32768 +# this parameter is multiplied by the exponential backoff when doing a liveness check so the first will be 800, then 1600, then 3200 etc... +pastry_socket_scm_ping_delay = 800 +# adds some fuzziness to the pings to help prevent congestion collapse, so this will make the ping be advanced or delayed by this factor +pastry_socket_scm_ping_jitter = 0.1 +# how many pings until we call the node faulty +pastry_socket_scm_num_ping_tries = 5 +pastry_socket_scm_write_wait_time = 30000 +pastry_socket_scm_backoff_initial = 250 +pastry_socket_scm_backoff_limit = 5 +pastry_socket_pingmanager_testSourceRouting = false +pastry_socket_increment_port_after_construction = true +# if you want to allow connection to 127.0.0.1, set this to true +pastry_socket_allow_loopback = false +# these params will be used if the computer attempts to bind to the loopback address, they will open a socket to this address/port to identify which network adapter to bind to +pastry_socket_known_network_address = yahoo.com +pastry_socket_known_network_address_port = 80 +pastry_socket_use_own_random = true +pastry_socket_random_seed = clock +# force the node to be a seed node +rice_socket_seed = false + +# the parameter simulates some nodes being firewalled, base on rendezvous_test_num_firewalled +rendezvous_test_firewall = false +# probabilistic fraction of firewalled nodes +rendezvous_test_num_firewalled = 0.3 +# don't firewall the first node, useful for testing +rendezvous_test_makes_bootstrap = false + +# FP 2.1 uses the new transport layer +transport_wire_datagram_receive_buffer_size = 131072 +transport_wire_datagram_send_buffer_size = 65536 + +# NAT/UPnP settings +nat_network_prefixes = 127.0.0.1;10.;192.168. +# Enable and set this if you have already set up port forwarding and know the external address +#external_address = 123.45.67.89:1234 +#enable this if you set up port forwarding (on the same port), but you don't +#know the external address and you don't have UPnP enabled +#this is useful for a firwall w/o UPnP support, and your IP address isn't static +probe_for_external_address = false +# values how to probe +pastry_proxy_connectivity_timeout = 15000 +pastry_proxy_connectivity_tries = 3 +# possible values: always, never, prefix (prefix is if the localAddress matches any of the nat_network_prefixes +# whether to search for a nat using UPnP (default: prefix) +nat_search_policy = prefix +# whether to verify connectivity (default: boot) +firewall_test_policy = boot +# policy for setting port forwarding the state of the firewall if there is already a conflicting rule: overwrite, fail (throw exception), change (use different port) +# you may want to set this to overwrite or fail on the bootstrap nodes, but most freepastry applications can run on any available port, so the default is change +nat_state_policy = change +# the name of the application in the firewall, set this if you want your application to have a more specific name +nat_app_name = freepastry +# how long to wait for responses from the firewall, in millis +nat_discovery_timeout = 5000 +# how many searches to try to find a free firewall port +nat_find_port_max_tries = 10 +# uncomment this to use UPnP NAT port forwarding, you need to include in the classpath: commons-jxpath-1.1.jar:commons-logging.jar:sbbi-upnplib-xxx.jar +nat_handler_class = rice.pastry.socket.nat.sbbi.SBBINatHandler +# hairpinning: +# default "prefix" requires more bandwidth if you are behind a NAT. It enables multiple IP +# addresses in the NodeHandle if you are behind a NAT. These are usually the internet routable address, +# and the LAN address (usually 192.168.x.x) +# you can set this to never if any of the following conditions hold: +# a) you are the only FreePastry node behind this address +# b) you firewall supports hairpinning see +# http://scm.sipfoundry.org/rep/ietf-drafts/behave/draft-ietf-behave-nat-udp-0... +nat_nodehandle_multiaddress = prefix + +# if we are not scheduled for time on cpu in this time, we setReady(false) +# otherwise there could be message inconsistency, because +# neighbors may believe us to be dead. Note that it is critical +# to consider the amount of time it takes the transport layer to find a +# node faulty before setting this parameter, this parameter should be +# less than the minimum time required to find a node faulty +pastry_protocol_consistentJoin_max_time_to_be_scheduled = 15000 + +# in case messages are dropped or something, how often it will retry to +# send the consistent join message, to get verification from the entire +# leafset +pastry_protocol_consistentJoin_retry_interval = 30000 +# parameter to control how long dead nodes are retained in the "failed set" in +# CJP (see ConsistentJoinProtocol ctor) (15 minutes) +pastry_protocol_consistentJoin_failedRetentionTime = 900000 +# how often to cleanup the failed set (5 mins) (see ConsistentJoinProtocol ctor) +pastry_protocol_consistentJoin_cleanup_interval = 300000 +# the maximum number of entries to send in the failed set, only sends the most +recent detected failures (see ConsistentJoinProtocol ctor) +pastry_protocol_consistentJoin_maxFailedToSend = 20 + +# how often we send/expect to be sent updates +pastry_protocol_periodicLeafSet_ping_neighbor_period = 20000 +pastry_protocol_periodicLeafSet_lease_period = 30000 + +# what the grace period is to receive a periodic update, before checking +# liveness +pastry_protocol_periodicLeafSet_request_lease_throttle = 10000 + +# how many entries are kept in the partition handler's table +partition_handler_max_history_size=20 +# how long entries in the partition handler's table are kept +# 90 minutes +partition_handler_max_history_age=5400000 +# what fraction of the time a bootstrap host is checked +partition_handler_bootstrap_check_rate=0.05 +# how often to run the partition handler +# 5 minutes +partition_handler_check_interval=300000 + +# the version number of the RouteMessage to transmit (it can receive anything that it knows how to) +# this is useful if you need to migrate an older ring +# you can change this value in realtime, so, you can start at 0 and issue a command to update it to 1 +pastry_protocol_router_routeMsgVersion = 1 + +# should usually be equal to the pastry_rtBaseBitLength +p2p_splitStream_stripeBaseBitLength = 4 +p2p_splitStream_policy_default_maximum_children = 24 +p2p_splitStream_stripe_max_failed_subscription = 5 +p2p_splitStream_stripe_max_failed_subscription_retry_delay = 1000 + +#multiring +p2p_multiring_base = 2 + +#past +p2p_past_messageTimeout = 30000 +p2p_past_successfulInsertThreshold = 0.5 + +#replication + +# fetch delay is the delay between fetching successive keys +p2p_replication_manager_fetch_delay = 500 +# the timeout delay is how long we take before we time out fetching a key +p2p_replication_manager_timeout_delay = 20000 +# this is the number of keys to delete when we detect a change in the replica set +p2p_replication_manager_num_delete_at_once = 100 +# this is how often replication will wake up and do maintainence; 10 mins +p2p_replication_maintenance_interval = 600000 +# the maximum number of keys replication will try to exchange in a maintainence message +p2p_replication_max_keys_in_message = 1000 + +#scribe +p2p_scribe_maintenance_interval = 180000 +#time for a subscribe fail to be thrown (in millis) +p2p_scribe_message_timeout = 15000 + +#util +p2p_util_encryptedOutputStream_buffer = 32678 + +#aggregation +p2p_aggregation_logStatistics = true +p2p_aggregation_flushDelayAfterJoin = 30000 +#5 MINS +p2p_aggregation_flushStressInterval = 300000 +#5 MINS +p2p_aggregation_flushInterval = 300000 +#1024*1024 +p2p_aggregation_maxAggregateSize = 1048576 +p2p_aggregation_maxObjectsInAggregate = 25 +p2p_aggregation_maxAggregatesPerRun = 2 +p2p_aggregation_addMissingAfterRefresh = true +p2p_aggregation_maxReaggregationPerRefresh = 100 +p2p_aggregation_nominalReferenceCount = 2 +p2p_aggregation_maxPointersPerAggregate = 100 +#14 DAYS +p2p_aggregation_pointerArrayLifetime = 1209600000 +#1 DAY +p2p_aggregation_aggregateGracePeriod = 86400000 +#15 MINS +p2p_aggregation_aggrRefreshInterval = 900000 +p2p_aggregation_aggrRefreshDelayAfterJoin = 70000 +#3 DAYS +p2p_aggregation_expirationRenewThreshold = 259200000 +p2p_aggregation_monitorEnabled = false +#15 MINS +p2p_aggregation_monitorRefreshInterval = 900000 +#5 MINS +p2p_aggregation_consolidationDelayAfterJoin = 300000 +#15 MINS +p2p_aggregation_consolidationInterval = 900000 +#14 DAYS +p2p_aggregation_consolidationThreshold = 1209600000 +p2p_aggregation_consolidationMinObjectsInAggregate = 20 +p2p_aggregation_consolidationMinComponentsAlive = 0.8 +p2p_aggregation_reconstructionMaxConcurrentLookups = 10 +p2p_aggregation_aggregateLogEnabled = true +#1 HOUR +p2p_aggregation_statsGranularity = 3600000 +#3 WEEKS +p2p_aggregation_statsRange = 1814400000 +p2p_aggregation_statsInterval = 60000 +p2p_aggregation_jitterRange = 0.1 + +# glacier +p2p_glacier_logStatistics = true +p2p_glacier_faultInjectionEnabled = false +p2p_glacier_insertTimeout = 30000 +p2p_glacier_minFragmentsAfterInsert = 3.0 +p2p_glacier_refreshTimeout = 30000 +p2p_glacier_expireNeighborsDelayAfterJoin = 30000 +#5 MINS +p2p_glacier_expireNeighborsInterval = 300000 +#5 DAYS +p2p_glacier_neighborTimeout = 432000000 +p2p_glacier_syncDelayAfterJoin = 30000 +#5 MINS +p2p_glacier_syncMinRemainingLifetime = 300000 +#insertTimeout +p2p_glacier_syncMinQuietTime = 30000 +p2p_glacier_syncBloomFilterNumHashes = 3 +p2p_glacier_syncBloomFilterBitsPerKey = 4 +p2p_glacier_syncPartnersPerTrial = 1 +#1 HOUR +p2p_glacier_syncInterval = 3600000 +#3 MINUTES +p2p_glacier_syncRetryInterval = 180000 +p2p_glacier_syncMaxFragments = 100 +p2p_glacier_fragmentRequestMaxAttempts = 0 +p2p_glacier_fragmentRequestTimeoutDefault = 10000 +p2p_glacier_fragmentRequestTimeoutMin = 10000 +p2p_glacier_fragmentRequestTimeoutMax = 60000 +p2p_glacier_fragmentRequestTimeoutDecrement = 1000 +p2p_glacier_manifestRequestTimeout = 10000 +p2p_glacier_manifestRequestInitialBurst = 3 +p2p_glacier_manifestRequestRetryBurst = 5 +p2p_glacier_manifestAggregationFactor = 5 +#3 MINUTES +p2p_glacier_overallRestoreTimeout = 180000 +p2p_glacier_handoffDelayAfterJoin = 45000 +#4 MINUTES +p2p_glacier_handoffInterval = 240000 +p2p_glacier_handoffMaxFragments = 10 +#10 MINUTES +p2p_glacier_garbageCollectionInterval = 600000 +p2p_glacier_garbageCollectionMaxFragmentsPerRun = 100 +#10 MINUTES +p2p_glacier_localScanInterval = 600000 +p2p_glacier_localScanMaxFragmentsPerRun = 20 +p2p_glacier_restoreMaxRequestFactor = 4.0 +p2p_glacier_restoreMaxBoosts = 2 +p2p_glacier_rateLimitedCheckInterval = 30000 +p2p_glacier_rateLimitedRequestsPerSecond = 3 +p2p_glacier_enableBulkRefresh = true +p2p_glacier_bulkRefreshProbeInterval = 3000 +p2p_glacier_bulkRefreshMaxProbeFactor = 3.0 +p2p_glacier_bulkRefreshManifestInterval = 30000 +p2p_glacier_bulkRefreshManifestAggregationFactor = 20 +p2p_glacier_bulkRefreshPatchAggregationFactor = 50 +#3 MINUTES +p2p_glacier_bulkRefreshPatchInterval = 180000 +p2p_glacier_bulkRefreshPatchRetries = 2 +p2p_glacier_bucketTokensPerSecond = 100000 +p2p_glacier_bucketMaxBurstSize = 200000 +p2p_glacier_jitterRange = 0.1 +#1 MINUTE +p2p_glacier_statisticsReportInterval = 60000 +p2p_glacier_maxActiveRestores = 3 + +#transport layer testing params +org.mpisws.p2p.testing.transportlayer.replay.Recorder_printlog = true + +# logging +#default log level +loglevel = WARNING + +#example of enabling logging on the endpoint: +#rice.p2p.scribe@ScribeRegrTest-endpoint_loglevel = INFO + +logging_packageOnly = true + +logging_date_format = yyyyMMdd.HHmmss.SSS +logging_enable=true + +# 24 hours +log_rotate_interval = 86400000 +# the name of the active log file, and the filename prefix of rotated log +log_rotate_filename = freepastry.log +# the format of the date for the rotating log +log_rotating_date_format = yyyyMMdd.HHmmss.SSS + +# true will tell the environment to ues the FileLogManager +environment_logToFile = false +# the prefix for the log files (otherwise will be named after the nodeId) +fileLogManager_filePrefix = +# the suffix for the log files +fileLogManager_fileSuffix = .log +# wether to keep the line prefix (declaring the node id) for each line of the log +fileLogManager_keepLinePrefix = false +fileLogManager_multipleFiles = true +fileLogManager_defaultFileName = main + +# false = append true = overwrite +fileLogManager_overwrite_existing_log_file = false + +# the amount of time the LookupService tutorial app will wait before timing out +# in milliseconds, default is 30 seconds +lookup_service.timeout = 30000 +# how long to wait before the first retry +lookup_service.firstTimeout = 500 Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-02 12:56:07 UTC (rev 55) @@ -5,4 +5,5 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=WARN \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=INFO +log4j.logger.org.nuiton.disworkfs.Demo=INFO \ No newline at end of file Added: trunk/diswork-fs/src/main/resources/tester.properties =================================================================== --- trunk/diswork-fs/src/main/resources/tester.properties (rev 0) +++ trunk/diswork-fs/src/main/resources/tester.properties 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,11 @@ +# IP where the coordinator will execute +tester.server=127.0.0.1 + +# path and the name for the peerunit logfile +tester.logfile=peerunit.log + +# path to store the logfiles of the peers +tester.logfolder=/tmp/peerunit + +# number of peer that execute the test +tester.peers=2 Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,19 @@ +package org.nuiton.disworkfs; + +import org.junit.Before; + +public class DisworkFileSystemInMemoryTest extends DisworkFileSystemTest { + + /** + * this code executed after {@link DisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); + disworkConfig.setOption("diswork.fs.use_in_memory_map", "true"); + fileSystem = new DisworkFileSystem(disworkConfig); + } + +} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,82 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Before; +import org.junit.Test; + +public class DisworkFileSystemPastryTest extends DisworkFileSystemTest { + + /** + * this code executed after {@link DisworkFileSystemTest#setUp()} + * @throws Exception + */ + + private static final Log log = LogFactory.getLog(DisworkFileSystemPastryTest.class); + + protected Integer bootstrapPort; + + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = Util.newDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + + } + + /* + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + log.info("----------------------trying to create second directory"); + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + */ + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -4,6 +4,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.File; @@ -11,6 +12,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ConcurrentModificationException; import java.util.List; import java.util.Random; @@ -22,7 +24,7 @@ import org.nuiton.util.FileUtil; -public class DisworkFileSystemTest { +public abstract class DisworkFileSystemTest { /** * a place to store files for the test it's a subdirectory of the OS temp @@ -66,9 +68,6 @@ File randomFile = new File(randomFilePath); FileUtils.writeByteArrayToFile(randomFile, randomBytes); - // finally, initiate the fileSystem - DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); - fileSystem = new DisworkFileSystem(disworkConfig); } @After @@ -311,6 +310,18 @@ // ... and only those assertEquals(3, lsResult.size()); + + lsResult = fileSystem.readDirectory("/"); + assertEquals(1, lsResult.size()); } - + + @Test + public void testConcurrency() throws Exception { + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + } } Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,159 +0,0 @@ -/* *##% - * Copyright (c) 2010 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.nuiton.disworkfs; - - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; -import org.nuiton.disworkfs.storage.EntryUtil; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class EntryUtilTest { - - static protected String directoryEntry = - EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "mydir" + EntryUtil.ENTRY_SEPARATOR - + "myid1"; - - static protected String fileEntry = - EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR - + "myfile" + EntryUtil.ENTRY_SEPARATOR - + "myid2"; - - static protected String linkEntry = - EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR - + "mylink" + EntryUtil.ENTRY_SEPARATOR - + "myid3"; - - static protected String metaBlock = - "12345" + EntryUtil.BLOCKIDS_SEPARATOR - + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR - + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR - + "myid3"; - - @Test - public void testIsDirectory() { - assertTrue(EntryUtil.isDirectory(directoryEntry)); - assertFalse(EntryUtil.isDirectory(fileEntry)); - } - - @Test - public void testFind() { - - String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR - + fileEntry + EntryUtil.ENTRIES_SEPARATOR - + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - - // seek and return entry - String findResult; - - findResult = EntryUtil.findEntryInDirectory(content, "mydir"); - assertEquals(directoryEntry, findResult); - findResult = EntryUtil.findEntryInDirectory(content, "myfile"); - assertEquals(fileEntry, findResult); - findResult = EntryUtil.findEntryInDirectory(content, "mylink"); - assertEquals(linkEntry, findResult); - - - // return null if file not found - findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); - assertEquals(null, findResult); - } - - @Test - public void testGetId() { - assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry)); - assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry)); - assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry)); - } - - @Test - public void testGetName() { - assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry)); - assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry)); - assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry)); - - } - - @Test - public void testGetType() { - assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry)); - assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry)); - assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry)); - } - - @Test - public void testResolveLink() { - String path; - - path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); - assertEquals("/anotherdir", path); - path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); - assertEquals("/dir/subdir/anotherdir", path); - path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); - assertEquals("/dir/anotherdir", path); - path = EntryUtil.resolveLink("/dir/", "subdir/file"); - assertEquals("/dir/subdir/file", path); - } - - @Test - public void testBytesToArray() { - String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$"; - String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); - assertEquals(s, copy); - } - - @Test - public void testGetTotalSizeFromMetaBlock() { - assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); - } - - @Test - public void testGetBlockIdsFromMetaBlock() { - String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; - String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); - assertArrayEquals(expectedBlocksIds, actualBlocksIds); - } - - @Test - public void testRemoveEntryFromEntries() { - - String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR - + fileEntry + EntryUtil.ENTRIES_SEPARATOR - + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - - String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR - + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - - String actual = EntryUtil.removeEntryFromEntries(content, "myfile"); - - assertEquals(expected, actual); - - } -} Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-06-02 10:27:22 UTC (rev 54) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -1,27 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertFalse; - -import java.util.Arrays; - -import org.junit.Test; -import org.nuiton.disworkfs.storage.InMemoryMap; - -public class InMemoryMapTest { - - /** - * this test show that the InMemoryMap put implies a copy of the data. - * After the put, the original value is modified. A final check shows - * there is no side effect - */ - @Test - public void testPut() { - InMemoryMap map = new InMemoryMap(); - byte[] expected = {0x1, 0x2}; - map.put("key", expected); - expected[0] = 0xf; - byte[] actual = map.get("key"); - assertFalse(Arrays.equals(expected, actual)); - } - -} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,89 @@ +package org.nuiton.disworkfs; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + +/** + * This class provides utilities to write tests easily. + * + * You can get multiples diswork configs ready to use. All the instances uses + * a new port and the local IP. + * <pre> + * c = newDisworkConfig(); // create a config for a bootstrap node + * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that + * // will bootstrap by joining the + * // first node + * c3 = newDisworkConfig(c.getUsedPort()) + * </pre> + * + */ +public class Util { + + protected static Integer port = 9000; + + /** + * returns a new port, returned value change at each call. + * @return + */ + public static Integer getPort() { + port += 1; + return port; + } + + /** + * returns the IP on the local machine. Trying to get an public IP or a LAN + * IP or the loopback IP if there is no other interface + * @return + * @throws UnknownHostException + */ + public static InetAddress getIp() throws UnknownHostException { + InetAddress result = InetAddress.getLocalHost(); + if (result.isLoopbackAddress()) { + try { + Socket temp = new Socket("microsoft.com", 80); + result = temp.getLocalAddress(); + temp.close(); + } catch (IOException e) { + // TODO 20100602 bleny do something ? + } + } + return result; + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a single-node instance of DisworkFS + * @return + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newDisworkConfig() + throws UnknownHostException { + return newDisworkConfig(null); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a multiple-node instance of DisworkFS on a same machine. + * @param bootstrapPort the port on the same machine where another node can + * be found to bootstrap + * @return a complete config + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = Util.getPort().toString(); + String ip = getIp().getHostAddress(); + result.setOption("diswork.fs.use_in_memory_map", "false"); + result.setOption("diswork.fs.use_port", port); + result.setOption("diswork.fs.bootstrap.ip", ip); + if (bootstrapPort == null) { + result.setOption("diswork.fs.bootstrap.port", port); + } else { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + } + return result; + } +} \ No newline at end of file Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java (from rev 52, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,159 @@ +/* *##% + * Copyright (c) 2010 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.nuiton.disworkfs.storage; + + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.nuiton.disworkfs.storage.EntryUtil; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class EntryUtilTest { + + static protected String directoryEntry = + EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "mydir" + EntryUtil.ENTRY_SEPARATOR + + "myid1"; + + static protected String fileEntry = + EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR + + "myfile" + EntryUtil.ENTRY_SEPARATOR + + "myid2"; + + static protected String linkEntry = + EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR + + "mylink" + EntryUtil.ENTRY_SEPARATOR + + "myid3"; + + static protected String metaBlock = + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid3"; + + @Test + public void testIsDirectory() { + assertTrue(EntryUtil.isDirectory(directoryEntry)); + assertFalse(EntryUtil.isDirectory(fileEntry)); + } + + @Test + public void testFind() { + + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + // seek and return entry + String findResult; + + findResult = EntryUtil.findEntryInDirectory(content, "mydir"); + assertEquals(directoryEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "mylink"); + assertEquals(linkEntry, findResult); + + + // return null if file not found + findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, findResult); + } + + @Test + public void testGetId() { + assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry)); + assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry)); + assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry)); + } + + @Test + public void testGetName() { + assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry)); + assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry)); + assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry)); + + } + + @Test + public void testGetType() { + assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry)); + assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry)); + assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry)); + } + + @Test + public void testResolveLink() { + String path; + + path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); + assertEquals("/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); + assertEquals("/dir/subdir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); + assertEquals("/dir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/", "subdir/file"); + assertEquals("/dir/subdir/file", path); + } + + @Test + public void testBytesToArray() { + String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$"; + String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); + assertEquals(s, copy); + } + + @Test + public void testGetTotalSizeFromMetaBlock() { + assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); + } + + @Test + public void testGetBlockIdsFromMetaBlock() { + String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; + String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); + assertArrayEquals(expectedBlocksIds, actualBlocksIds); + } + + @Test + public void testRemoveEntryFromEntries() { + + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + String actual = EntryUtil.removeEntryFromEntries(content, "myfile"); + + assertEquals(expected, actual); + + } +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java (from rev 52, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,27 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertFalse; + +import java.util.Arrays; + +import org.junit.Test; +import org.nuiton.disworkfs.storage.InMemoryDisworkMap; + +public class InMemoryMapTest { + + /** + * this test show that the InMemoryMap put implies a copy of the data. + * After the put, the original value is modified. A final check shows + * there is no side effect + */ + @Test + public void testPut() { + InMemoryDisworkMap map = new InMemoryDisworkMap(); + byte[] expected = {0x1, 0x2}; + map.put("key", expected); + expected[0] = 0xf; + byte[] actual = map.get("key"); + assertFalse(Arrays.equals(expected, actual)); + } + +} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-02 12:56:07 UTC (rev 55) @@ -0,0 +1,98 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.disworkfs.Util; + +public class PastryDisworkMapTest { + + protected static PastryDisworkMap map1; + protected static PastryDisworkMap map2; + + protected static byte[] bytes; + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + /* + if (map1 != null) + map1.close(); + if (map2 != null) + map2.close(); + */ + } + + @Test + public void test() throws Exception { + DisworkFileSystemConfig config = Util.newDisworkConfig(); + + map1 = new PastryDisworkMap(config); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + bytes = newBytes; + + map1.put("test", newBytes); + + byte[] getResult = map1.get("test"); + assertArrayEquals(newBytes, getResult); + } + + @Test + public void test2nodes() throws Exception { + DisworkFileSystemConfig config1 = Util.newDisworkConfig(); + map1 = new PastryDisworkMap(config1); + + DisworkFileSystemConfig config2 = Util.newDisworkConfig(config1.getUsedPort()); + map2 = new PastryDisworkMap(config2); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + bytes = newBytes; + + map1.put("test", newBytes); + + assertTrue(map2.containsKey("test")); + assertFalse(map2.containsKey("unknownkey")); + + byte[] getResult = map2.get("test"); + + assertArrayEquals(newBytes, getResult); + + byte[] removeResult = map2.remove("test"); + assertArrayEquals(newBytes, removeResult); + + map1.put("test", newBytes); + map2.put("test", newBytes); + + map2.remove("unknow"); + + } + + @Test + public void testClose() throws Exception { + /* + DisworkFileSystemConfig config1 = new DisworkFileSystemConfig(); + map1 = new PastryDisworkMap(config1); + */ + /* + try { + map2 = new PastryDisworkMap(config1); + } catch (IllegalStateException e) { + assertEquals("Cannot bind to /192.168.99.119:9001", e.getMessage()); + } + */ + + // map1.close(); + // map2 = new PastryDisworkMap(config1); + } + +}