Diswork-commits
Threads by month
- ----- 2026 -----
- June
- May
- April
- March
- February
- January
- ----- 2025 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2010 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- 136 discussions
03 Jun '10
Author: bleny
Date: 2010-06-03 12:23:17 +0200 (Thu, 03 Jun 2010)
New Revision: 57
Url: http://nuiton.org/repositories/revision/diswork/57
Log:
remove bad import
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-02 13:08:02 UTC (rev 56)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 10:23:17 UTC (rev 57)
@@ -2,17 +2,12 @@
import java.io.IOException;
import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.util.List;
import java.util.Random;
-import javax.swing.plaf.SliderUI;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.junit.runner.Runner;
public class Demo {
@@ -92,8 +87,7 @@
} catch (InterruptedException e) {
log.info("sleep interrupted", e);
}
-
- }
+ }
}
}
1
0
02 Jun '10
Author: bleny
Date: 2010-06-02 15:08:02 +0200 (Wed, 02 Jun 2010)
New Revision: 56
Url: http://nuiton.org/repositories/revision/diswork/56
Log:
cleaning pom, some documentation
Added:
trunk/src/site/rst/diswork-fs/how_it_works.rst
Modified:
trunk/diswork-fs/pom.xml
trunk/pom.xml
trunk/src/site/site_en.xml
Modified: trunk/diswork-fs/pom.xml
===================================================================
--- trunk/diswork-fs/pom.xml 2010-06-02 12:56:07 UTC (rev 55)
+++ trunk/diswork-fs/pom.xml 2010-06-02 13:08:02 UTC (rev 56)
@@ -1,62 +1,48 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.nuiton</groupId>
- <artifactId>diswork</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.nuiton</groupId>
+ <artifactId>diswork</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
- <groupId>org.nuiton.diswork</groupId>
- <artifactId>diswork-fs</artifactId>
+ <groupId>org.nuiton.diswork</groupId>
+ <artifactId>diswork-fs</artifactId>
- <packaging>jar</packaging>
- <name>disworkfs</name>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.0.2</version>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.nuiton</groupId>
- <artifactId>nuiton-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.freepastry</groupId>
- <artifactId>pastry</artifactId>
- </dependency>
- <dependency>
- <groupId>fr.inria.peerunit</groupId>
- <artifactId>PeerUnit</artifactId>
- </dependency>
+ <packaging>jar</packaging>
+ <name>disworkfs</name>
+ <dependencies>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.nuiton</groupId>
+ <artifactId>nuiton-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.freepastry</groupId>
+ <artifactId>pastry</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>fr.inria.peerunit</groupId>
+ <artifactId>PeerUnit</artifactId>
+ </dependency>
- <!-- test -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
-
+ <!-- test -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
Modified: trunk/pom.xml
===================================================================
--- trunk/pom.xml 2010-06-02 12:56:07 UTC (rev 55)
+++ trunk/pom.xml 2010-06-02 13:08:02 UTC (rev 56)
@@ -1,11 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+ <modelVersion>4.0.0</modelVersion>
- <!-- ************************************************************* -->
- <!-- *** POM Relationships *************************************** -->
- <!-- ************************************************************* -->
+ <!-- ************************************************************* -->
+ <!-- *** POM Relationships *************************************** -->
+ <!-- ************************************************************* -->
<parent>
<groupId>org.nuiton</groupId>
@@ -13,176 +14,183 @@
<version>2.1.5</version>
</parent>
- <artifactId>diswork</artifactId>
- <version>0.0.1-SNAPSHOT</version>
+ <artifactId>diswork</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
- <modules>
- <module>diswork-fs</module>
- </modules>
+ <modules>
+ <module>diswork-fs</module>
+ </modules>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- <version>1.1.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.14</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>jgroups</groupId>
- <artifactId>jgroups</artifactId>
- <version>2.9.0.GA</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>1.4</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.5</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.nuiton</groupId>
- <artifactId>nuiton-utils</artifactId>
- <version>1.2.2</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-<!--
- <dependency>
- <groupId>fr.inria.peerunit</groupId>
- <artifactId>PeerUnit</artifactId>
- <version>1.0</version>
- <scope>compile</scope>
- </dependency>
--->
- <!-- test -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.8.1</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
- <!-- ************************************************************* -->
- <!-- *** Project Information ************************************* -->
- <!-- ************************************************************* -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.14</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>jgroups</groupId>
+ <artifactId>jgroups</artifactId>
+ <version>2.9.0.GA</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>1.4</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.nuiton</groupId>
+ <artifactId>nuiton-utils</artifactId>
+ <version>1.2.2</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>fr.inria.peerunit</groupId>
+ <artifactId>PeerUnit</artifactId>
+ <version>1.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.kth</groupId>
+ <artifactId>dks</artifactId>
+ <version>0.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.freepastry</groupId>
+ <artifactId>pastry</artifactId>
+ <version>2.1</version>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ <!-- ************************************************************* -->
+ <!-- *** Project Information ************************************* -->
+ <!-- ************************************************************* -->
- <name>Diswork</name>
- <description>Distributed computing application</description>
- <inceptionYear>2010</inceptionYear>
- <url>http://maven-site.nuiton.org/diswork</url>
+ <name>Diswork</name>
+ <description>Distributed computing application</description>
+ <inceptionYear>2010</inceptionYear>
+ <url>http://maven-site.nuiton.org/diswork</url>
- <developers>
- <developer>
- <name>Brendan Le Ny</name>
- <id>bleny</id>
- <email>leny(a)codelutin.com</email>
- <organization>CodeLutin</organization>
- <timezone>+2</timezone>
- <roles>
- <role>Développeur</role>
- </roles>
- </developer>
- <developer>
- <name>Benjamin Poussin</name>
- <id>bpoussin</id>
- <email>poussin(a)codelutin.com</email>
- <organization>CodeLutin</organization>
- <timezone>+2</timezone>
- <roles>
- <role>Développeur</role>
- <role>Debian packager</role>
- </roles>
- </developer>
- <developer>
- <name>Eric Chatellier</name>
- <id>echatellier</id>
- <email>chatellier(a)codelutin.com</email>
- <organization>CodeLutin</organization>
- <timezone>+2</timezone>
- <roles>
- <role>Développeur</role>
- </roles>
- </developer>
- <developer>
- <name>Jean Couteau</name>
- <id>jcouteau</id>
- <email>couteau(a)codelutin.com</email>
- <organization>CodeLutin</organization>
- <timezone>+2</timezone>
- <roles>
- <role>Développeur</role>
- </roles>
- </developer>
- <developer>
- <name>Tony Chemit</name>
- <id>tchemit</id>
- <email>chemit(a)codelutin.com</email>
- <organization>CodeLutin</organization>
- <timezone>+2</timezone>
- <roles>
- <role>Développeur</role>
- </roles>
- </developer>
- </developers>
+ <developers>
+ <developer>
+ <name>Brendan Le Ny</name>
+ <id>bleny</id>
+ <email>leny(a)codelutin.com</email>
+ <organization>CodeLutin</organization>
+ <timezone>+2</timezone>
+ <roles>
+ <role>Développeur</role>
+ </roles>
+ </developer>
+ <developer>
+ <name>Benjamin Poussin</name>
+ <id>bpoussin</id>
+ <email>poussin(a)codelutin.com</email>
+ <organization>CodeLutin</organization>
+ <timezone>+2</timezone>
+ <roles>
+ <role>Développeur</role>
+ <role>Debian packager</role>
+ </roles>
+ </developer>
+ <developer>
+ <name>Eric Chatellier</name>
+ <id>echatellier</id>
+ <email>chatellier(a)codelutin.com</email>
+ <organization>CodeLutin</organization>
+ <timezone>+2</timezone>
+ <roles>
+ <role>Développeur</role>
+ </roles>
+ </developer>
+ <developer>
+ <name>Jean Couteau</name>
+ <id>jcouteau</id>
+ <email>couteau(a)codelutin.com</email>
+ <organization>CodeLutin</organization>
+ <timezone>+2</timezone>
+ <roles>
+ <role>Développeur</role>
+ </roles>
+ </developer>
+ <developer>
+ <name>Tony Chemit</name>
+ <id>tchemit</id>
+ <email>chemit(a)codelutin.com</email>
+ <organization>CodeLutin</organization>
+ <timezone>+2</timezone>
+ <roles>
+ <role>Développeur</role>
+ </roles>
+ </developer>
+ </developers>
- <!-- ************************************************************* -->
- <!-- *** Build Settings ****************************************** -->
- <!-- ************************************************************* -->
+ <!-- ************************************************************* -->
+ <!-- *** Build Settings ****************************************** -->
+ <!-- ************************************************************* -->
- <packaging>pom</packaging>
+ <packaging>pom</packaging>
- <properties>
- <!-- name of redmine project -->
- <projectId>diswork</projectId>
-
- <!-- locales to use in documentation -->
- <locales>en</locales>
- </properties>
+ <properties>
+ <!-- name of redmine project -->
+ <projectId>diswork</projectId>
- <build>
-
+ <!-- locales to use in documentation -->
+ <locales>en</locales>
+ </properties>
- <pluginManagement>
- <plugins>
- <plugin>
- <artifactId>maven-site-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.nuiton.jrst</groupId>
- <artifactId>doxia-module-jrst</artifactId>
- <version>${jrst.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.nuiton.jrst</groupId>
+ <artifactId>doxia-module-jrst</artifactId>
+ <version>${jrst.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
- <!-- ************************************************************* -->
- <!-- *** Build Environment ************************************** -->
- <!-- ************************************************************* -->
+ <!-- ************************************************************* -->
+ <!-- *** Build Environment ************************************** -->
+ <!-- ************************************************************* -->
- <!-- Source control management. -->
- <scm>
- <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection>
- <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection>
- <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url>
- </scm>
+ <!-- Source control management. -->
+ <scm>
+ <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection>
+ <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection>
+ <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url>
+ </scm>
</project>
Added: trunk/src/site/rst/diswork-fs/how_it_works.rst
===================================================================
--- trunk/src/site/rst/diswork-fs/how_it_works.rst (rev 0)
+++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-02 13:08:02 UTC (rev 56)
@@ -0,0 +1,125 @@
+How Diswork File System works
+=============================
+
+Diswork File System uses a map to store data persistently.
+
+We illustrate the use of a file system by setting the content of "/",
+the root directory as follow:
+
+* a directory named "my_folder" that contains:
+
+ - a directory named "my_sub_folder" that contains:
+
+ + a file name "my_deep_file.ext"
+
+ + an empty folder named "my_empty_folder"
+
+ - a file named "my_other_file.ext"
+
+* a file named "my_file.ext" (containing binary data: 010101)
+
+* a symbolic link named "my_link" which target the path
+ "/my_folder/my_sub_folder/my_deep_file.ext"
+
+Storage uses a Map to store data.
+
++--------------+---------------------------------------------+
+| Key (String) | Value (byte[]) |
++==============+=============================================+
+| | "D:my_folder:ID1\n |
+| | F:my_file.ext:ID2\n |
+| "/" | L:my_link:ID3" |
++--------------+---------------------------------------------+
+| | "D:my_sub_folder:ID4\n |
+| "ID1" | F:my_other_file.ext:ID5" |
++--------------+---------------------------------------------+
+| "ID2" | 010101 |
++--------------+---------------------------------------------+
+| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" |
++--------------+---------------------------------------------+
+| | "F:my_deep_file:ID6\n |
+| "ID4" | D:my_empty_folder:ID7" |
++--------------+---------------------------------------------+
+| "ID5" | 011010100111010... |
++--------------+---------------------------------------------+
+| "ID6" | 1100010101010110110010... |
++--------------+---------------------------------------------+
+| "ID7" | "" |
++--------------+---------------------------------------------+
+
+An entry is, for example, "D:my_folder:ID1". It contains the type of the
+entry (<strong>D</strong>irectory, <strong>F</strong>ile or
+<strong>L</strong>ink), the name of the element, and an ID to be used as
+a key on the map to get the actual content. Those three informations are
+separated by ":" which is EntryUtil.ENTRY_SEPARATOR.
+
+A directory way have multiple <em>entries</em>. Entries of a directory are
+separated by "\n" which is EntryUtil.ENTRIES_SEPARATOR.
+
+The above description shows the main principle used to store a tree
+structure in a map.
+
+In fact, the storage of actual data is a bit more complex, due to
+the need of splitting the data that may be too large (the value of "ID6"
+may be a series of 1002<sup>20</sup> bytes for a 200 MiB file).
+Actually, files content (like ID2, ID5, ID6) and directory content
+values ("/", ID1, ID4) will be split and the map will look like that (
+<strong>note that only the files have been split for readability</strong>):
+
++--------------+---------------------------------------------+
+| Key (String) | Value (byte[]) |
++==============+=============================================+
+| | "D:my_folder:ID1\n |
+| | F:my_file.ext:ID2\n |
+| "/" | L:my_link:ID3" |
++--------------+---------------------------------------------+
+| | "D:my_sub_folder:ID4\n |
+| "ID1" | F:my_other_file.ext:ID5" |
++--------------+---------------------------------------------+
+| "ID2" | "6;ID8" |
++--------------+---------------------------------------------+
+| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" |
++--------------+---------------------------------------------+
+| | "F:my_deep_file:ID6\n |
+| "ID4" | D:my_empty_folder:ID7" |
++--------------+---------------------------------------------+
+| "ID5" | "14689;ID9" |
++--------------+---------------------------------------------+
+| "ID6" | "79874567;ID10;ID11;ID12" |
++--------------+---------------------------------------------+
+| "ID7" | "" |
++--------------+---------------------------------------------+
+| "ID8" | 010101 |
++--------------+---------------------------------------------+
+| "ID9" | 011010100111010... |
++--------------+---------------------------------------------+
+| "ID10" | 110001010101011... |
++--------------+---------------------------------------------+
+| "ID11" | 111010110101101... |
++--------------+---------------------------------------------+
+| "ID12" | 011101100111010... |
++--------------+---------------------------------------------+
+
+Values of ID2, ID5, ID6 are no longer the file content. It's now a set
+of meta-information information data called metablock.
+A metablock is composed of the total size of the file followed by
+an ordered lists of IDs of the different blocks composing the file.
+Those informations are separated by ";" (see
+{@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}).
+
+In the above example:
+
+* "/my_file.ext" is a file which size is 6 bytes, it is
+ composed of one block available at ID8;
+
+* "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes)
+ which content is available at ID9;
+
+* "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its
+ size is 79874567. It has been split in three blocks: ID10, ID11 and
+ ID12.
+
+When reading and writing in storage, split is done transparently. When
+reading, a Stream is returned: it loads data blocks after blocks
+when needed inner class Storage.SplitBlocksInputStream}).
+When writing, data are split in blocks of a maximum configurable size.
\ No newline at end of file
Modified: trunk/src/site/site_en.xml
===================================================================
--- trunk/src/site/site_en.xml 2010-06-02 12:56:07 UTC (rev 55)
+++ trunk/src/site/site_en.xml 2010-06-02 13:08:02 UTC (rev 56)
@@ -29,6 +29,7 @@
<menu name="diswork FS">
<item href="diswork-fs/history.html" name="Install"/>
+ <item href="diswork-fs/how_it_works.html" name="How it works"/>
</menu>
<menu ref="reports"/>
1
0
r55 - in trunk/diswork-fs: . src/main/java/org/nuiton/disworkfs src/main/java/org/nuiton/disworkfs/peerunit src/main/java/org/nuiton/disworkfs/storage src/main/resources src/test/java/org/nuiton/disworkfs src/test/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org 02 Jun '10
by bleny＠users.nuiton.org 02 Jun '10
02 Jun '10
Author: bleny
Date: 2010-06-02 14:56:07 +0200 (Wed, 02 Jun 2010)
New Revision: 55
Url: http://nuiton.org/repositories/revision/diswork/55
Log:
implementation of DisworkMap with FreePastry ; more documentation
Added:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java
trunk/diswork-fs/src/main/resources/freepastry.params
trunk/diswork-fs/src/main/resources/tester.properties
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java
Removed:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java
Modified:
trunk/diswork-fs/pom.xml
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java
trunk/diswork-fs/src/main/resources/log4j.properties
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java
Modified: trunk/diswork-fs/pom.xml
===================================================================
--- trunk/diswork-fs/pom.xml 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/pom.xml 2010-06-02 12:56:07 UTC (rev 55)
@@ -1,4 +1,5 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.nuiton</groupId>
@@ -11,20 +12,20 @@
<packaging>jar</packaging>
<name>disworkfs</name>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.0.2</version>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
@@ -41,12 +42,15 @@
<groupId>org.nuiton</groupId>
<artifactId>nuiton-utils</artifactId>
</dependency>
-<!--
<dependency>
+ <groupId>org.freepastry</groupId>
+ <artifactId>pastry</artifactId>
+ </dependency>
+ <dependency>
<groupId>fr.inria.peerunit</groupId>
<artifactId>PeerUnit</artifactId>
</dependency>
--->
+
<!-- test -->
<dependency>
<groupId>junit</groupId>
Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,221 @@
+package org.nuiton.disworkfs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Random;
+
+import javax.swing.plaf.SliderUI;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.runner.Runner;
+
+public class Demo {
+
+ private static final Log log = LogFactory.getLog(Demo.class);
+
+ protected static DisworkFileSystem fileSystem;
+
+ protected static class Producer implements Runnable {
+
+ protected String waitingForJob = null;
+ protected Integer expectedResult = null;
+
+ @Override
+ public void run() {
+ while (true) {
+
+ try {
+ if (fileSystem.exists("/todo")) {
+
+ if (waitingForJob == null) {
+
+ // taking a random job
+ Random random = new Random();
+
+ waitingForJob = "/todo/job-" + random.nextInt();
+ Integer randomInteger = random.nextInt();
+ expectedResult = randomInteger + 1;
+ String job = randomInteger.toString();
+
+ InputStream source = IOUtils.toInputStream(job);
+
+
+ log.info("writing job " + waitingForJob + " (" + randomInteger + ")");
+
+ fileSystem.write(waitingForJob, source);
+
+ source.close();
+
+ List<String> todos = fileSystem.readDirectory("/todo");
+
+ System.out.println("number of jobs : " + todos.size());
+
+ } else {
+ String resultPath = waitingForJob + ".result";
+ if (fileSystem.exists(resultPath)) {
+ log.info("results has been published");
+
+ InputStream resultStream = fileSystem.read(resultPath);
+
+ String resultString = IOUtils.toString(resultStream);
+
+ Integer result = Integer.parseInt(resultString);
+
+ log.info("result is " + result + "(expected : " + expectedResult + ")");
+
+ fileSystem.delete(resultPath);
+ fileSystem.delete(waitingForJob);
+
+ waitingForJob = null;
+ expectedResult = null;
+
+ }
+
+ }
+
+
+ } else {
+ fileSystem.createDirectory("/todo");
+ }
+
+ } catch (IOException e) {
+ log.error(e);
+ }
+
+ try {
+ Thread.sleep(15 * 1000);
+ } catch (InterruptedException e) {
+ log.info("sleep interrupted", e);
+ }
+
+ }
+ }
+ }
+
+ protected static class Consumer implements Runnable {
+
+ @Override
+ public void run() {
+ while (true) {
+
+ try {
+ if (fileSystem.exists("/todo")) {
+
+ List<String> todos = fileSystem.readDirectory("/todo");
+
+ if (todos.isEmpty()) {
+ log.info("nothing to do");
+ } else {
+ // taking a random job
+ Random random = new Random();
+ String todo = todos.get(random.nextInt(todos.size()));
+
+ if (!todo.endsWith(".result")) {
+ String jobPath = "/todo/" + todo;
+ log.info("reading the job " + jobPath);
+
+ InputStream in = fileSystem.read(jobPath);
+
+ String operation = IOUtils.toString(in);
+
+ // in.close();
+
+ log.info("operation to do " + operation);
+
+ Integer i = Integer.parseInt(operation);
+
+ i += 1;
+
+ String result = i.toString();
+
+ log.info("result is " + result);
+
+ InputStream source = IOUtils.toInputStream(result);
+
+ fileSystem.write(jobPath + ".result", source);
+
+ source.close();
+
+ }
+ }
+
+ } else {
+ fileSystem.createDirectory("/todo");
+ }
+
+ } catch (IOException e) {
+ log.error(e);
+ }
+
+ try {
+ Thread.sleep(10 * 1000);
+ } catch (InterruptedException e) {
+ log.info("sleep interrupted", e);
+ }
+
+ }
+ }
+ }
+
+ public static void printUsage() {
+ System.out.println(
+ "Usage :\n"
+ + "org.nuiton.disworkfs.Demo producer|consumer usePort [bootStrapIp bootStrapPort]"
+ );
+ }
+
+ /**
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+
+ if (args.length == 2) {
+
+ DisworkFileSystemConfig config = new DisworkFileSystemConfig();
+
+ config.setOption("diswork.fs.use_port", args[1]);
+
+ fileSystem = new DisworkFileSystem(config);
+
+ if ("producer".equals(args[0])) {
+ Thread t = new Thread(new Producer());
+ log.info("starting a producer");
+ t.start();
+ } else if ("consumer".equals(args[0])) {
+ Thread t = new Thread(new Consumer());
+ log.info("starting a consumer");
+ t.start();
+ }
+
+ } else if (args.length == 4) {
+
+ DisworkFileSystemConfig config = new DisworkFileSystemConfig();
+
+ config.setOption("diswork.fs.use_port", args[1]);
+ config.setOption("diswork.fs.bootstrap.ip", args[2]);
+ config.setOption("diswork.fs.bootstrap.port", args[3]);
+
+ fileSystem = new DisworkFileSystem(config);
+
+ if ("producer".equals(args[2])) {
+ Thread t = new Thread(new Producer());
+ log.info("starting a producer");
+ t.start();
+ } else if ("consumer".equals(args[2])) {
+ Thread t = new Thread(new Consumer());
+ log.info("starting a consumer");
+ t.start();
+ }
+
+ } else {
+ printUsage();
+ }
+ }
+
+}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -11,13 +11,12 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.nuiton.disworkfs.storage.EntryUtil;
-import org.nuiton.disworkfs.storage.InMemoryMap;
import org.nuiton.disworkfs.storage.Storage;
/** Main class of Diswork File System, provide methods for all operations
* You can use:
* <ul>
- * <li>{@link #createDirectory(String)} and {@link DisworkFileSystem#readDirectory(String)}
+ * <li>{@link #createDirectory(String)} and {@link #readDirectory(String)}
* to create and browse directories</li>
* <li>{@link #write(String, InputStream)} and {@link #read(String)}
* to write a file and read it</li>
@@ -44,13 +43,12 @@
/** constructor allowing to provide another configuration to use
*
- * @param disworkConfig the configuration
+ * @param config the configuration
* @throws IOException
*/
- public DisworkFileSystem(DisworkFileSystemConfig disworkConfig)
+ public DisworkFileSystem(DisworkFileSystemConfig config)
throws IOException {
- storage = new Storage(disworkConfig, new InMemoryMap());
- storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent());
+ storage = new Storage(config);
}
/** tests the existence of a file/dir/link at a given path
@@ -98,6 +96,9 @@
String id = EntryUtil.getIdFromEntry(entry);
result = storage.getFile(id);
}
+
+ log.info("read " + path + " returns " + result.available() + " bytes");
+
return result;
}
@@ -116,6 +117,7 @@
}
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
+ log.info("writing " + source.available() + " bytes at " + path);
write(parent, name, source);
}
@@ -186,6 +188,7 @@
// store file before meta info
storage.putDirectory(newDirectoryId,
EntryUtil.newEmptyDirectoryContent());
+
// update meta info directory
storage.putDirectory(parentId, newContent);
} else {
@@ -329,18 +332,25 @@
entry = walk(target);
}
if (EntryUtil.isDirectory(entry)) {
+ result = new ArrayList<String>();
String content = storage.getDirectory(
EntryUtil.getIdFromEntry(entry));
- String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR);
- result = new ArrayList<String>();
- for (String elementEntry : entries) {
- result.add(EntryUtil.getNameFromEntry(elementEntry));
+ if (EntryUtil.newEmptyDirectoryContent().equals(content)) {
+ // directory is empty, add nothing
+ } else {
+ String[] entries = content.split(
+ EntryUtil.ENTRIES_SEPARATOR);
+ for (String elementEntry : entries) {
+ result.add(EntryUtil.getNameFromEntry(elementEntry));
+ }
}
} else {
throw new IOException(path + " is not a directory");
}
}
+ log.info("readDirectory " + path + " returns " + result.size() + " results");
+
return result;
}
@@ -418,7 +428,7 @@
log.info("in intermediate dir " + current + ", looking for " + p);
// mise a jour de current
- if (current.equals("/")) current = "";
+ if (current.equals("/")) current = ""; // avoid "//path" next line
current += "/" + p;
String entry = EntryUtil.findEntryInDirectory(content, p);
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -15,12 +15,32 @@
* </dl>
*/
public class DisworkFileSystemConfig extends ApplicationConfig {
-
+
public DisworkFileSystemConfig() {
- setDefaultOption("blocks_size", "10485760"); // 10 MiB
+ setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB
+ setDefaultOption("diswork.fs.use_in_memory_map", "false");
+ setDefaultOption("diswork.fs.use_port", "9001");
+ setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119");
+ setDefaultOption("diswork.fs.bootstrap.port", "9001");
}
public int getBlockSize() {
- return getOptionAsInt("blocks_size");
+ return getOptionAsInt("diswork.fs.blocks_size");
}
+
+ public boolean useInMemoryMap() {
+ return getOptionAsBoolean("diswork.fs.use_in_memory_map");
+ }
+
+ public Integer getUsedPort() {
+ return getOptionAsInt("diswork.fs.use_port");
+ }
+
+ public String getBootstrapIp() {
+ return getOption("diswork.fs.bootstrap.ip");
+ }
+
+ public Integer getBootstrapPort() {
+ return getOptionAsInt("diswork.fs.bootstrap.port");
+ }
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -1,10 +1,11 @@
/**
* DisworkFS is a distributed file system. You can use it by instantiating
- * {@link DisworkFileSystem} and use it to store directories and their content.
+ * {@link org.nuiton.disworkfs.DisworkFileSystem} and use it to store
+ * directories and their content.
*
* You can change the default DisworkFileSystem behavior by provide a
- * {@link DisworkFileSystemConfig} instance at construction. If you don't
- * provide one, a instance will be created
+ * {@link org.nuiton.disworkfs.DisworkFileSystemConfig} instance at
+ * construction. If you don't provide one, a instance will be created
*/
package org.nuiton.disworkfs;
\ No newline at end of file
Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,93 @@
+package org.nuiton.disworkfs.peerunit;
+
+import static fr.inria.peerunit.test.assertion.Assert.assertTrue;
+import static fr.inria.peerunit.test.assertion.Assert.assertEquals;
+import static fr.inria.peerunit.test.assertion.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import org.apache.commons.io.IOUtils;
+import org.nuiton.disworkfs.DisworkFileSystem;
+import org.nuiton.disworkfs.DisworkFileSystemConfig;
+
+import fr.inria.peerunit.TestCaseImpl;
+import fr.inria.peerunit.parser.TestStep;
+
+public class DisworkFileSystemTest extends TestCaseImpl {
+
+ protected Integer port = 9001;
+
+ protected DisworkFileSystem fileSystem;
+
+ protected byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+
+ @TestStep(range="0",timeout=1000000, order = 0)
+ public void testConnect() throws Exception {
+ Integer myPort = port + getId();
+ DisworkFileSystemConfig config = new DisworkFileSystemConfig();
+ config.setOption("diswork.fs.use_port", myPort.toString());
+ fileSystem = new DisworkFileSystem(config);
+ }
+
+ @TestStep(range="1",timeout=1000000, order = 1)
+ public void testConnect1() throws Exception {
+ Integer myPort = port + getId();
+ DisworkFileSystemConfig config = new DisworkFileSystemConfig();
+ config.setOption("diswork.fs.use_port", myPort.toString());
+ fileSystem = new DisworkFileSystem(config);
+ }
+ /*
+ @TestStep(range="0",timeout=1000000, order = 2)
+ public void testWrite() throws Exception {
+ InputStream source = new ByteArrayInputStream(bytes);
+ fileSystem.write("/myfile", source);
+ source.close();
+ }
+
+ @TestStep(range="1",timeout=1000000, order = 3)
+ public void testRead() throws Exception {
+ try {
+ InputStream source = fileSystem.read("/myfile");
+ byte[] readResult = IOUtils.toByteArray(source);
+ source.close();
+ boolean arraysContentEquals = Arrays.equals(bytes, readResult);
+ assertTrue(arraysContentEquals);
+ } catch (FileNotFoundException e) {
+ fail();
+ }
+ }
+ */
+ @TestStep(range="1",timeout=1000000, order = 4)
+ public void testCreateDir() throws Exception {
+ fileSystem.createDirectory("/mydir");
+ fileSystem.createDirectory("/myseconddir");
+ }
+
+ @TestStep(range="0-1",timeout=1000000, order = 5)
+ public void fake() throws Exception {
+
+ }
+
+ /*
+ @TestStep(range="0-1",timeout=1000000, order = 5)
+ public void testCreateDir2() throws Exception {
+ assertTrue(fileSystem.exists("/mydir"));
+ assertTrue(fileSystem.exists("/myseconddir"));
+ }
+ */
+ /*
+ @TestStep(range="1",timeout=1000000, order = 6)
+ public void testCreateSubDir() throws Exception {
+ fileSystem.createDirectory("/mydir/mysubdir");
+ }
+
+ @TestStep(range="1",timeout=1000000, order = 7)
+ public void testReadDir() throws Exception {
+ assertTrue(fileSystem.exists("/mydir/mysubdir"));
+ assertEquals(1, fileSystem.readDirectory("/mydir").size());
+ }
+ */
+}
Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,6 @@
+package org.nuiton.disworkfs.storage;
+
+import java.io.Closeable;
+import java.util.Map;
+
+public interface DisworkMap extends Map<String, byte[]>, Closeable {}
Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java (from rev 52, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java)
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,32 @@
+package org.nuiton.disworkfs.storage;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/** An in-memory implementation of a Map
+ * This class is a map where put make a copy of the value. It is used for
+ * stubbing purpose. It emulates the behavior of a DHT and thus can be used
+ * by {@link Storage} to store data and blocks.
+ */
+public class InMemoryDisworkMap extends HashMap<String, byte[]>
+ implements DisworkMap {
+
+ private final Log log = LogFactory.getLog(InMemoryDisworkMap.class);
+
+ @Override
+ public byte[] put(String key, byte[] value) {
+ log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")");
+ byte[] valueCopy = value.clone();
+ log.info("putting " + valueCopy.length + " bytes");
+ return super.put(key, valueCopy);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // nothing to do
+ }
+}
\ No newline at end of file
Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -1,25 +0,0 @@
-package org.nuiton.disworkfs.storage;
-
-import java.util.Arrays;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/** An in-memory implementation of a Map
- * This class is a map where put make a copy of the value. It is used for
- * stubbing purpose. It emulates the behavior of a DHT and thus can be used
- * by {@link Storage} to store data and blocks.
- */
-public class InMemoryMap extends HashMap<String, byte[]> {
-
- private final Log log = LogFactory.getLog(InMemoryMap.class);
-
- @Override
- public byte[] put(String key, byte[] value) {
- log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")");
- byte[] valueCopy = value.clone();
- log.info("putting " + valueCopy.length + " bytes");
- return super.put(key, valueCopy);
- }
-}
\ No newline at end of file
Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,363 @@
+package org.nuiton.disworkfs.storage;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.nuiton.disworkfs.DisworkFileSystemConfig;
+
+import rice.Continuation;
+import rice.environment.Environment;
+import rice.p2p.commonapi.Id;
+import rice.p2p.past.ContentHashPastContent;
+import rice.p2p.past.PastContent;
+import rice.p2p.past.PastImpl;
+import rice.pastry.NodeIdFactory;
+import rice.pastry.PastryNode;
+import rice.pastry.PastryNodeFactory;
+import rice.pastry.commonapi.PastryIdFactory;
+import rice.pastry.socket.SocketPastryNodeFactory;
+import rice.pastry.standard.RandomNodeIdFactory;
+import rice.persistence.EmptyCache;
+import rice.persistence.MemoryStorage;
+import rice.persistence.StorageManagerImpl;
+
+/**
+ * An implementation of {@link DisworkMap} using FreePastry.
+ *
+ * @see <a href="https://trac.freepastry.org/">FreePastry site</a>
+ *
+ */
+public class PastryDisworkMap implements DisworkMap {
+
+ private final Log log = LogFactory.getLog(PastryDisworkMap.class);
+
+ protected static class ByteArrayPastContent extends ContentHashPastContent {
+
+ private static final long serialVersionUID = 7584834640788361309L;
+
+ byte[] content;
+
+ public ByteArrayPastContent(Id id, byte[] content) {
+ super(id);
+ this.content = content;
+ }
+
+ public String toString() {
+ return "ByteArrayPastContent (" + content.length + " bytes)";
+ }
+ }
+
+ protected abstract class MyContinuation<R, E extends Exception> implements Continuation<R, Exception> {
+
+ protected boolean finished = false;
+
+ public boolean isFinished() {
+ return finished;
+ }
+
+ public void waitUntilFinised() {
+ while(!finished) {
+ try {
+ Thread.sleep(1500);
+ log.info("waiting for response");
+ } catch (InterruptedException e) {
+ // TODO 20100527 bleny stub
+ log.info("exception catch", e);
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public void receiveException(Exception exception) {
+ log.error("exception received : " + exception);
+ finished = true;
+ }
+
+ @Override
+ public void receiveResult(R result) {
+ finished = true;
+ }
+
+ }
+
+ protected PastImpl past;
+
+ protected PastryIdFactory pastryIdFactory;
+
+ public PastryDisworkMap(DisworkFileSystemConfig config) throws IOException {
+
+ Environment env = new Environment();
+
+ // disable the UPnP setting (in case you are testing this on a NATted LAN)
+ // env.getParameters().setString("nat_search_policy", "never");
+
+ // the port to use locally
+ int bindport = config.getUsedPort();
+
+ InetAddress bootaddr = InetAddress.getByName(config.getBootstrapIp());
+ int bootport = config.getBootstrapPort();
+
+ InetSocketAddress bootaddress = new InetSocketAddress(bootaddr, bootport);
+
+ log.info("boot address : " + bootaddress);
+
+ NodeIdFactory nidFactory = new RandomNodeIdFactory(env);
+
+ // construct the PastryNodeFactory, this is how we use rice.pastry.socket
+ PastryNodeFactory factory = null;
+
+ // FIXME 20100602 bleny this is a work-around to deal with time out on
+ // connect exceptions that occurs "sometimes"
+ int numberOfTry = 0;
+ while(factory == null && numberOfTry <= 10) {
+ try {
+ factory = new SocketPastryNodeFactory(nidFactory, bindport, env);
+ } catch (java.net.ConnectException e) {
+ numberOfTry += 1;
+ }
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ log.info("exception catch", e);
+ e.printStackTrace();
+ }
+ }
+ if (factory == null) {
+ throw new IOException("unable to connect");
+ }
+
+ // construct a node, but this does not cause it to boot
+ PastryNode node = factory.newNode();
+
+ // in later tutorials, we will register applications before calling boot
+ node.boot(bootaddress);
+
+ // the node may require sending several messages to fully boot into the ring
+ synchronized(node) {
+ while(!node.isReady() && !node.joinFailed()) {
+ // delay so we don't busy-wait
+ try {
+ node.wait(500);
+ } catch (InterruptedException e) {
+ // TODO 20100528 bleny Auto-generated catch block
+ log.info("exception catch", e);
+ e.printStackTrace();
+ }
+
+ // abort if can't join
+ if (node.joinFailed()) {
+ throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason());
+ }
+ }
+ }
+
+ log.info("finished creating new node " + node);
+
+ pastryIdFactory = new PastryIdFactory(env);
+
+ rice.persistence.Storage storage = new MemoryStorage(pastryIdFactory);
+ past = new PastImpl(
+ node,
+ new StorageManagerImpl(
+ pastryIdFactory,
+ storage,
+ // Using a cache do not permit to remove while putting
+ /*
+ new LRUCache(
+ new MemoryStorage(pastryIdFactory),
+ 512 * 1024,
+ node.getEnvironment()
+ )
+ */
+ new EmptyCache(pastryIdFactory)
+ ),
+ 0, // replication factor
+ ""
+ );
+
+
+ }
+
+ protected class ContainsKeyContinuation extends MyContinuation<Boolean, Exception> {
+
+ public Boolean result = null;
+
+ @Override
+ public void receiveResult(Boolean result) {
+ log.info("contains key result received : " + result);
+ this.result = result;
+ finished = true;
+ }
+ }
+
+ @Override
+ public boolean containsKey(Object key) {
+ Id id = pastryIdFactory.buildId((String) key);
+ ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation();
+ past.existsInOverlay(id, containsKeyContinuation);
+ containsKeyContinuation.waitUntilFinised();
+ boolean result = containsKeyContinuation.result;
+ log.info("containsKey " + key + " (id = " + id + ") returns " + result);
+ return result;
+ }
+
+ protected class GetContinuation extends MyContinuation<PastContent, Exception> {
+
+ public byte[] result = null;
+
+ @Override
+ public void receiveResult(PastContent result) {
+ log.info("get result received " + result);
+ if (result != null) {
+ this.result = ((ByteArrayPastContent) result).content;
+ }
+ finished = true;
+ }
+ }
+
+ protected byte[] atomicGet(Object key) {
+ Id id = pastryIdFactory.buildId((String) key);
+ GetContinuation getContinuation = new GetContinuation();
+ past.lookup(id, getContinuation);
+ getContinuation.waitUntilFinised();
+ return getContinuation.result;
+ }
+
+ @Override
+ public byte[] get(Object key) {
+ byte[] result = atomicGet(key);
+ if (result == null) {
+ log.info("get " + key + " returns null");
+ } else {
+ log.info("get " + key + " returns " + result.length + " bytes");
+ }
+ return result;
+ }
+
+ protected class PutContinuation extends MyContinuation<Boolean[], Exception> {
+ @Override
+ public void receiveResult(Boolean[] result) {
+ log.info("insert result received : " + Arrays.toString(result));
+ finished = true;
+ }
+ }
+
+ protected void atomicPut(String key, byte[] value) {
+ Id id = pastryIdFactory.buildId(key);
+ PastContent pastContent = new ByteArrayPastContent(id, value);
+ PutContinuation putContinuation = new PutContinuation();
+ past.insert(pastContent, putContinuation);
+ putContinuation.waitUntilFinised();
+ }
+
+ @Override
+ public byte[] put(String key, byte[] value) {
+ byte[] previousValue = atomicGet(key);
+ if (previousValue != null) {
+ atomicRemove(key);
+ }
+ atomicPut(key, value);
+ log.info("put " + value.length + " bytes at " + key );
+ return previousValue;
+ }
+
+ protected class RemoveContinuation extends MyContinuation<Boolean, Exception> {
+
+ public Boolean result = null;
+
+ @Override
+ public void receiveResult(Boolean result) {
+ log.info("remove result received : " + result);
+ this.result = result;
+ finished = true;
+ }
+ }
+
+ protected void atomicRemove(Object key) {
+ Id id = pastryIdFactory.buildId((String) key);
+ RemoveContinuation removeContinuation = new RemoveContinuation();
+ past.remove(id, removeContinuation);
+ removeContinuation.waitUntilFinised();
+ log.info("atomic remove " + key + " has returned " + removeContinuation.result);
+ }
+
+ @Override
+ public byte[] remove(Object key) {
+ byte[] previousValue = atomicGet(key);
+ if (previousValue == null) {
+ log.info("remove " + key + " returns null");
+ } else {
+ atomicRemove(key);
+ log.info("remove " + key + " returns " + previousValue.length + " bytes");
+ }
+ return previousValue;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // TODO 20100528 bleny should close Pastry and close all sockets
+ // throw new UnsupportedOperationException("not yet implemented");
+ /*
+ ServerSocket s = new ServerSocket(9001);
+ s.getChannel().close();
+ s.close();
+ */
+ }
+
+ @Deprecated
+ @Override
+ public void putAll(Map<? extends String, ? extends byte[]> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public int size() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public Collection<byte[]> values() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public boolean isEmpty() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public Set<String> keySet() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public boolean containsValue(Object arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ @Override
+ public Set<java.util.Map.Entry<String, byte[]>> entrySet() {
+ throw new UnsupportedOperationException();
+ }
+
+}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -15,7 +15,11 @@
import org.nuiton.disworkfs.DisworkFileSystemConfig;
/**
+ * This class is the middle layer between the File System operations and
+ * the Map<String, byte[]>. It offers put and get on directories, files
+ * and links and deal with splitting consideration and concurrency
*
+ * It needs a place to store data called a {@link DisworkMap}.
*/
public class Storage implements Closeable {
@@ -104,13 +108,23 @@
}
- protected Map<String, byte[]> map;
+ protected DisworkMap map;
protected DisworkFileSystemConfig disworkConfig;
- public Storage(DisworkFileSystemConfig disworkConfig, Map<String, byte[]> map) {
- this.map = map;
+ public Storage(DisworkFileSystemConfig disworkConfig) throws IOException {
this.disworkConfig = disworkConfig;
+ if (disworkConfig.useInMemoryMap()) {
+ log.info("using in-memory map");
+ this.map = new InMemoryDisworkMap();
+ putDirectory("/", EntryUtil.newEmptyDirectoryContent());
+ } else {
+ log.info("using Pastry map");
+ this.map = new PastryDisworkMap(disworkConfig);
+ if (! map.containsKey("/")) {
+ putDirectory("/", EntryUtil.newEmptyDirectoryContent());
+ }
+ }
}
/**
@@ -189,30 +203,36 @@
// trying to acquire lock
Long currentDate = System.currentTimeMillis();
- byte[] lock = map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString()));
+ log.info("trying to acquire a lock on " + key);
+ byte[] lock = map.put(lockKey,
+ EntryUtil.stringToBytes(currentDate.toString()));
if (lock != null) {
// file is locked, check date
Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock));
-
- if (currentDate - currentLock > LOCK_VALID_TIME) {
- // this lock is out-dated, let's erase it
+ Long lockAge = currentDate - currentLock;
+ if (lockAge > LOCK_VALID_TIME) {
+ log.info("lock is out-dated (" + lockAge + " ms old)");
+ // this lock is out-dated, let's erase all data
String obsoleteMetaBlock =
EntryUtil.bytesToString(map.get(newDataKey));
if (obsoleteMetaBlock != null) {
String[] obsoleteBlocksIds =
EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock);
+ log.info("removing " + obsoleteBlocksIds.length + " old blocks");
for (String obsoleteBlockId : obsoleteBlocksIds) {
map.remove(obsoleteBlockId);
}
}
} else {
- // file is currently written, stopping operation
+ log.info(key + " is currently written, stopping operation");
map.put(lockKey, lock);
throw new ConcurrentModificationException("can't write due to concurrency");
}
}
+ log.info("lock on " + key + " acquired");
+
// here, we know we can write or an exception
// would have been thrown earlier
String blocksIds = "";
@@ -258,6 +278,8 @@
}
map.put(key, map.get(newDataKey));
+
+ log.info("release lock " + lockKey);
map.remove(lockKey);
}
@@ -283,6 +305,7 @@
@Override
public void close() throws IOException {
clean();
+ map.close();
}
}
\ No newline at end of file
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -1,7 +1,8 @@
/**
* <p>
* This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a
- * way to persistently store data. This is done by the {@link Storage} class
+ * way to persistently store data. This is done by the
+ * {@link org.nuiton.disworkfs.storage.Storage} class
* which permit to store different type of data.
* </p>
* <p>
@@ -82,12 +83,14 @@
* entry (<strong>D</strong>irectory, <strong>F</strong>ile or
* <strong>L</strong>ink), the name of the element, and an ID to be used as
* a key on the map to get the actual content. Those three informations are
- * separated by ":" which is {@link EntryUtil#ENTRY_SEPARATOR}.
+ * separated by ":" which is
+ * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRY_SEPARATOR}.
* </p>
*
* <p>
* A directory way have multiple <em>entries</em>. Entries of a directory are
- * separated by "\n" which is {@link EntryUtil#ENTRIES_SEPARATOR}.
+ * separated by "\n" which is
+ * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRIES_SEPARATOR}.
* </p>
* <p>
* The above description shows the main principle used to store a tree
@@ -171,7 +174,7 @@
* A metablock is composed of the total size of the file followed by
* an ordered lists of IDs of the different blocks composing the file.
* Those informations are separated by ";" (see
- * {@link EntryUtil#BLOCKIDS_SEPARATOR}).
+ * {@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}).
* </p>
* <p>
* In the above example:
@@ -194,9 +197,10 @@
* <p>
* When reading and writing in storage, split is done transparently. When
* reading, a Stream is returned: it loads data blocks after blocks
- * when needed (see {@link Storage.SplitBlocksInputStream}). When writing,
- * data are split in blocks of a maximum configurable size
- * (see {@link org.nuiton.disworkfs.DisworkConfig}).
+ * when needed (see
+ * {@link org.nuiton.disworkfs.storage.Storage.SplitBlocksInputStream}).
+ * When writing, data are split in blocks of a maximum configurable size
+ * (see {@link org.nuiton.disworkfs.DisworkFileSystemConfig}).
* </p>
*/
Added: trunk/diswork-fs/src/main/resources/freepastry.params
===================================================================
--- trunk/diswork-fs/src/main/resources/freepastry.params (rev 0)
+++ trunk/diswork-fs/src/main/resources/freepastry.params 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,404 @@
+# this file holds the default values for pastry and it's applications
+# you do not need to modify the default.params file to override these values
+# instead you can use your own params file to set values to override the
+# defaults. You can specify this file by constructing your
+# rice.environment.Environment() with the filename you wish to use
+# typically, you will want to be able to pass this file name from the command
+# line
+
+# max number of handles stored per routing table entry
+pastry_rtMax = 1
+pastry_rtBaseBitLength = 4
+
+# leafset size
+pastry_lSetSize = 24
+
+# maintenance frequencies
+pastry_leafSetMaintFreq = 60
+pastry_routeSetMaintFreq = 900
+
+# drop the message if pastry is not ready
+pastry_messageDispatch_bufferIfNotReady = false
+
+# number of messages to buffer while an app hasn't yet been registered
+pastry_messageDispatch_bufferSize = 32
+
+# FP 2.1 uses the new transport layer
+transport_wire_datagram_receive_buffer_size = 131072
+transport_wire_datagram_send_buffer_size = 65536
+transport_epoch_max_num_addresses = 2
+transport_sr_max_num_hops = 5
+
+# proximity neighbor selection
+transport_use_pns = true
+
+# number of rows in the routing table to consider during PNS
+# valid values are ALL, or a number
+pns_num_rows_to_use = 10
+
+# commonapi testing parameters
+
+# direct or socket
+commonapi_testing_exit_on_failure = true
+commonapi_testing_protocol = direct
+commonapi_testing_startPort = 5009
+commonapi_testing_num_nodes = 10
+# set this to specify the bootstrap node
+#commonapi_testing_bootstrap = localhost:5009
+
+# random number generator's seed, "CLOCK" uses the current clock time
+random_seed = CLOCK
+
+# sphere, euclidean or gt-itm
+direct_simulator_topology = sphere
+# -1 starts the simulation with the current time
+direct_simulator_start_time = -1
+#pastry_direct_use_own_random = true
+#pastry_periodic_leafset_protocol_use_own_random = true
+pastry_direct_gtitm_matrix_file=GNPINPUT
+# the number of stubs in your network
+pastry_direct_gtitm_max_overlay_size=1000
+# the number of virtual nodes at each stub: this allows you to simulate multiple "LANs" and allows cheeper scaling
+pastry_direct_gtitm_nodes_per_stub=1
+# the factor to multiply your file by to reach millis. Set this to 0.001 if your file is in microseconds. Set this to 1000 if your file is in seconds.
+pastry_direct_gtitm_delay_factor=1.0
+#millis of the maximum network delay for the generated network topologies
+pastry_direct_max_diameter=200
+pastry_direct_min_delay=2
+#setting this to false will use the old protocols which are about 200 times as fast, but may cause routing inconsistency in a real network. Probably won't in a simulator because it will never be incorrect about liveness
+pastry_direct_guarantee_consistency=true
+
+# rice.pastry.socket parameters
+# tells the factory you intend to use multiple nodes
+# this causes the logger to prepend all entries with the nodeid
+pastry_factory_multipleNodes = true
+pastry_factory_selectorPerNode = false
+pastry_factory_processorPerNode = false
+# number of bootstap nodehandles to fetch in parallel
+pastry_factory_bootsInParallel = 1
+
+# the maximum size of a message
+pastry_socket_reader_selector_deserialization_max_size = 1000000
+# the maximum number of outgoing messages to queue when a socket is slower than the number of messages you are queuing
+pastry_socket_writer_max_queue_length = 30
+pastry_socket_writer_max_msg_size = 20480
+pastry_socket_repeater_buffer_size = 65536
+pastry_socket_pingmanager_smallPings=true
+pastry_socket_pingmanager_datagram_receive_buffer_size = 131072
+pastry_socket_pingmanager_datagram_send_buffer_size = 65536
+# the time before it will retry a route that was already found dead
+pastry_socket_srm_check_dead_throttle = 300000
+pastry_socket_srm_proximity_timeout = 3600000
+pastry_socket_srm_ping_throttle = 30000
+pastry_socket_srm_default_rto = 3000
+pastry_socket_srm_rto_ubound = 10000
+pastry_socket_srm_rto_lbound = 50
+pastry_socket_srm_gain_h = 0.25
+pastry_socket_srm_gain_g = 0.125
+pastry_socket_scm_max_open_sockets = 300
+pastry_socket_scm_max_open_source_routes = 30
+# the maximum number of source routes to attempt, setting this to 0 will
+# effectively eliminate source route attempts
+# setting higher than the leafset does no good, it will be bounded by the leafset
+# a larger number tries more source routes, which could give you a more accurate
+# determination, however, is more likely to lead to congestion collapse
+pastry_socket_srm_num_source_route_attempts = 8
+pastry_socket_scm_socket_buffer_size = 32768
+# this parameter is multiplied by the exponential backoff when doing a liveness check so the first will be 800, then 1600, then 3200 etc...
+pastry_socket_scm_ping_delay = 800
+# adds some fuzziness to the pings to help prevent congestion collapse, so this will make the ping be advanced or delayed by this factor
+pastry_socket_scm_ping_jitter = 0.1
+# how many pings until we call the node faulty
+pastry_socket_scm_num_ping_tries = 5
+pastry_socket_scm_write_wait_time = 30000
+pastry_socket_scm_backoff_initial = 250
+pastry_socket_scm_backoff_limit = 5
+pastry_socket_pingmanager_testSourceRouting = false
+pastry_socket_increment_port_after_construction = true
+# if you want to allow connection to 127.0.0.1, set this to true
+pastry_socket_allow_loopback = false
+# these params will be used if the computer attempts to bind to the loopback address, they will open a socket to this address/port to identify which network adapter to bind to
+pastry_socket_known_network_address = yahoo.com
+pastry_socket_known_network_address_port = 80
+pastry_socket_use_own_random = true
+pastry_socket_random_seed = clock
+# force the node to be a seed node
+rice_socket_seed = false
+
+# the parameter simulates some nodes being firewalled, base on rendezvous_test_num_firewalled
+rendezvous_test_firewall = false
+# probabilistic fraction of firewalled nodes
+rendezvous_test_num_firewalled = 0.3
+# don't firewall the first node, useful for testing
+rendezvous_test_makes_bootstrap = false
+
+# FP 2.1 uses the new transport layer
+transport_wire_datagram_receive_buffer_size = 131072
+transport_wire_datagram_send_buffer_size = 65536
+
+# NAT/UPnP settings
+nat_network_prefixes = 127.0.0.1;10.;192.168.
+# Enable and set this if you have already set up port forwarding and know the external address
+#external_address = 123.45.67.89:1234
+#enable this if you set up port forwarding (on the same port), but you don't
+#know the external address and you don't have UPnP enabled
+#this is useful for a firwall w/o UPnP support, and your IP address isn't static
+probe_for_external_address = false
+# values how to probe
+pastry_proxy_connectivity_timeout = 15000
+pastry_proxy_connectivity_tries = 3
+# possible values: always, never, prefix (prefix is if the localAddress matches any of the nat_network_prefixes
+# whether to search for a nat using UPnP (default: prefix)
+nat_search_policy = prefix
+# whether to verify connectivity (default: boot)
+firewall_test_policy = boot
+# policy for setting port forwarding the state of the firewall if there is already a conflicting rule: overwrite, fail (throw exception), change (use different port)
+# you may want to set this to overwrite or fail on the bootstrap nodes, but most freepastry applications can run on any available port, so the default is change
+nat_state_policy = change
+# the name of the application in the firewall, set this if you want your application to have a more specific name
+nat_app_name = freepastry
+# how long to wait for responses from the firewall, in millis
+nat_discovery_timeout = 5000
+# how many searches to try to find a free firewall port
+nat_find_port_max_tries = 10
+# uncomment this to use UPnP NAT port forwarding, you need to include in the classpath: commons-jxpath-1.1.jar:commons-logging.jar:sbbi-upnplib-xxx.jar
+nat_handler_class = rice.pastry.socket.nat.sbbi.SBBINatHandler
+# hairpinning:
+# default "prefix" requires more bandwidth if you are behind a NAT. It enables multiple IP
+# addresses in the NodeHandle if you are behind a NAT. These are usually the internet routable address,
+# and the LAN address (usually 192.168.x.x)
+# you can set this to never if any of the following conditions hold:
+# a) you are the only FreePastry node behind this address
+# b) you firewall supports hairpinning see
+# http://scm.sipfoundry.org/rep/ietf-drafts/behave/draft-ietf-behave-nat-udp-…
+nat_nodehandle_multiaddress = prefix
+
+# if we are not scheduled for time on cpu in this time, we setReady(false)
+# otherwise there could be message inconsistency, because
+# neighbors may believe us to be dead. Note that it is critical
+# to consider the amount of time it takes the transport layer to find a
+# node faulty before setting this parameter, this parameter should be
+# less than the minimum time required to find a node faulty
+pastry_protocol_consistentJoin_max_time_to_be_scheduled = 15000
+
+# in case messages are dropped or something, how often it will retry to
+# send the consistent join message, to get verification from the entire
+# leafset
+pastry_protocol_consistentJoin_retry_interval = 30000
+# parameter to control how long dead nodes are retained in the "failed set" in
+# CJP (see ConsistentJoinProtocol ctor) (15 minutes)
+pastry_protocol_consistentJoin_failedRetentionTime = 900000
+# how often to cleanup the failed set (5 mins) (see ConsistentJoinProtocol ctor)
+pastry_protocol_consistentJoin_cleanup_interval = 300000
+# the maximum number of entries to send in the failed set, only sends the most
+recent detected failures (see ConsistentJoinProtocol ctor)
+pastry_protocol_consistentJoin_maxFailedToSend = 20
+
+# how often we send/expect to be sent updates
+pastry_protocol_periodicLeafSet_ping_neighbor_period = 20000
+pastry_protocol_periodicLeafSet_lease_period = 30000
+
+# what the grace period is to receive a periodic update, before checking
+# liveness
+pastry_protocol_periodicLeafSet_request_lease_throttle = 10000
+
+# how many entries are kept in the partition handler's table
+partition_handler_max_history_size=20
+# how long entries in the partition handler's table are kept
+# 90 minutes
+partition_handler_max_history_age=5400000
+# what fraction of the time a bootstrap host is checked
+partition_handler_bootstrap_check_rate=0.05
+# how often to run the partition handler
+# 5 minutes
+partition_handler_check_interval=300000
+
+# the version number of the RouteMessage to transmit (it can receive anything that it knows how to)
+# this is useful if you need to migrate an older ring
+# you can change this value in realtime, so, you can start at 0 and issue a command to update it to 1
+pastry_protocol_router_routeMsgVersion = 1
+
+# should usually be equal to the pastry_rtBaseBitLength
+p2p_splitStream_stripeBaseBitLength = 4
+p2p_splitStream_policy_default_maximum_children = 24
+p2p_splitStream_stripe_max_failed_subscription = 5
+p2p_splitStream_stripe_max_failed_subscription_retry_delay = 1000
+
+#multiring
+p2p_multiring_base = 2
+
+#past
+p2p_past_messageTimeout = 30000
+p2p_past_successfulInsertThreshold = 0.5
+
+#replication
+
+# fetch delay is the delay between fetching successive keys
+p2p_replication_manager_fetch_delay = 500
+# the timeout delay is how long we take before we time out fetching a key
+p2p_replication_manager_timeout_delay = 20000
+# this is the number of keys to delete when we detect a change in the replica set
+p2p_replication_manager_num_delete_at_once = 100
+# this is how often replication will wake up and do maintainence; 10 mins
+p2p_replication_maintenance_interval = 600000
+# the maximum number of keys replication will try to exchange in a maintainence message
+p2p_replication_max_keys_in_message = 1000
+
+#scribe
+p2p_scribe_maintenance_interval = 180000
+#time for a subscribe fail to be thrown (in millis)
+p2p_scribe_message_timeout = 15000
+
+#util
+p2p_util_encryptedOutputStream_buffer = 32678
+
+#aggregation
+p2p_aggregation_logStatistics = true
+p2p_aggregation_flushDelayAfterJoin = 30000
+#5 MINS
+p2p_aggregation_flushStressInterval = 300000
+#5 MINS
+p2p_aggregation_flushInterval = 300000
+#1024*1024
+p2p_aggregation_maxAggregateSize = 1048576
+p2p_aggregation_maxObjectsInAggregate = 25
+p2p_aggregation_maxAggregatesPerRun = 2
+p2p_aggregation_addMissingAfterRefresh = true
+p2p_aggregation_maxReaggregationPerRefresh = 100
+p2p_aggregation_nominalReferenceCount = 2
+p2p_aggregation_maxPointersPerAggregate = 100
+#14 DAYS
+p2p_aggregation_pointerArrayLifetime = 1209600000
+#1 DAY
+p2p_aggregation_aggregateGracePeriod = 86400000
+#15 MINS
+p2p_aggregation_aggrRefreshInterval = 900000
+p2p_aggregation_aggrRefreshDelayAfterJoin = 70000
+#3 DAYS
+p2p_aggregation_expirationRenewThreshold = 259200000
+p2p_aggregation_monitorEnabled = false
+#15 MINS
+p2p_aggregation_monitorRefreshInterval = 900000
+#5 MINS
+p2p_aggregation_consolidationDelayAfterJoin = 300000
+#15 MINS
+p2p_aggregation_consolidationInterval = 900000
+#14 DAYS
+p2p_aggregation_consolidationThreshold = 1209600000
+p2p_aggregation_consolidationMinObjectsInAggregate = 20
+p2p_aggregation_consolidationMinComponentsAlive = 0.8
+p2p_aggregation_reconstructionMaxConcurrentLookups = 10
+p2p_aggregation_aggregateLogEnabled = true
+#1 HOUR
+p2p_aggregation_statsGranularity = 3600000
+#3 WEEKS
+p2p_aggregation_statsRange = 1814400000
+p2p_aggregation_statsInterval = 60000
+p2p_aggregation_jitterRange = 0.1
+
+# glacier
+p2p_glacier_logStatistics = true
+p2p_glacier_faultInjectionEnabled = false
+p2p_glacier_insertTimeout = 30000
+p2p_glacier_minFragmentsAfterInsert = 3.0
+p2p_glacier_refreshTimeout = 30000
+p2p_glacier_expireNeighborsDelayAfterJoin = 30000
+#5 MINS
+p2p_glacier_expireNeighborsInterval = 300000
+#5 DAYS
+p2p_glacier_neighborTimeout = 432000000
+p2p_glacier_syncDelayAfterJoin = 30000
+#5 MINS
+p2p_glacier_syncMinRemainingLifetime = 300000
+#insertTimeout
+p2p_glacier_syncMinQuietTime = 30000
+p2p_glacier_syncBloomFilterNumHashes = 3
+p2p_glacier_syncBloomFilterBitsPerKey = 4
+p2p_glacier_syncPartnersPerTrial = 1
+#1 HOUR
+p2p_glacier_syncInterval = 3600000
+#3 MINUTES
+p2p_glacier_syncRetryInterval = 180000
+p2p_glacier_syncMaxFragments = 100
+p2p_glacier_fragmentRequestMaxAttempts = 0
+p2p_glacier_fragmentRequestTimeoutDefault = 10000
+p2p_glacier_fragmentRequestTimeoutMin = 10000
+p2p_glacier_fragmentRequestTimeoutMax = 60000
+p2p_glacier_fragmentRequestTimeoutDecrement = 1000
+p2p_glacier_manifestRequestTimeout = 10000
+p2p_glacier_manifestRequestInitialBurst = 3
+p2p_glacier_manifestRequestRetryBurst = 5
+p2p_glacier_manifestAggregationFactor = 5
+#3 MINUTES
+p2p_glacier_overallRestoreTimeout = 180000
+p2p_glacier_handoffDelayAfterJoin = 45000
+#4 MINUTES
+p2p_glacier_handoffInterval = 240000
+p2p_glacier_handoffMaxFragments = 10
+#10 MINUTES
+p2p_glacier_garbageCollectionInterval = 600000
+p2p_glacier_garbageCollectionMaxFragmentsPerRun = 100
+#10 MINUTES
+p2p_glacier_localScanInterval = 600000
+p2p_glacier_localScanMaxFragmentsPerRun = 20
+p2p_glacier_restoreMaxRequestFactor = 4.0
+p2p_glacier_restoreMaxBoosts = 2
+p2p_glacier_rateLimitedCheckInterval = 30000
+p2p_glacier_rateLimitedRequestsPerSecond = 3
+p2p_glacier_enableBulkRefresh = true
+p2p_glacier_bulkRefreshProbeInterval = 3000
+p2p_glacier_bulkRefreshMaxProbeFactor = 3.0
+p2p_glacier_bulkRefreshManifestInterval = 30000
+p2p_glacier_bulkRefreshManifestAggregationFactor = 20
+p2p_glacier_bulkRefreshPatchAggregationFactor = 50
+#3 MINUTES
+p2p_glacier_bulkRefreshPatchInterval = 180000
+p2p_glacier_bulkRefreshPatchRetries = 2
+p2p_glacier_bucketTokensPerSecond = 100000
+p2p_glacier_bucketMaxBurstSize = 200000
+p2p_glacier_jitterRange = 0.1
+#1 MINUTE
+p2p_glacier_statisticsReportInterval = 60000
+p2p_glacier_maxActiveRestores = 3
+
+#transport layer testing params
+org.mpisws.p2p.testing.transportlayer.replay.Recorder_printlog = true
+
+# logging
+#default log level
+loglevel = WARNING
+
+#example of enabling logging on the endpoint:
+#rice.p2p.scribe@ScribeRegrTest-endpoint_loglevel = INFO
+
+logging_packageOnly = true
+
+logging_date_format = yyyyMMdd.HHmmss.SSS
+logging_enable=true
+
+# 24 hours
+log_rotate_interval = 86400000
+# the name of the active log file, and the filename prefix of rotated log
+log_rotate_filename = freepastry.log
+# the format of the date for the rotating log
+log_rotating_date_format = yyyyMMdd.HHmmss.SSS
+
+# true will tell the environment to ues the FileLogManager
+environment_logToFile = false
+# the prefix for the log files (otherwise will be named after the nodeId)
+fileLogManager_filePrefix =
+# the suffix for the log files
+fileLogManager_fileSuffix = .log
+# wether to keep the line prefix (declaring the node id) for each line of the log
+fileLogManager_keepLinePrefix = false
+fileLogManager_multipleFiles = true
+fileLogManager_defaultFileName = main
+
+# false = append true = overwrite
+fileLogManager_overwrite_existing_log_file = false
+
+# the amount of time the LookupService tutorial app will wait before timing out
+# in milliseconds, default is 30 seconds
+lookup_service.timeout = 30000
+# how long to wait before the first retry
+lookup_service.firstTimeout = 500
Modified: trunk/diswork-fs/src/main/resources/log4j.properties
===================================================================
--- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-02 12:56:07 UTC (rev 55)
@@ -5,4 +5,5 @@
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n
# package level
-log4j.logger.org.nuiton.disworkfs=WARN
\ No newline at end of file
+log4j.logger.org.nuiton.disworkfs=INFO
+log4j.logger.org.nuiton.disworkfs.Demo=INFO
\ No newline at end of file
Added: trunk/diswork-fs/src/main/resources/tester.properties
===================================================================
--- trunk/diswork-fs/src/main/resources/tester.properties (rev 0)
+++ trunk/diswork-fs/src/main/resources/tester.properties 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,11 @@
+# IP where the coordinator will execute
+tester.server=127.0.0.1
+
+# path and the name for the peerunit logfile
+tester.logfile=peerunit.log
+
+# path to store the logfiles of the peers
+tester.logfolder=/tmp/peerunit
+
+# number of peer that execute the test
+tester.peers=2
Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,19 @@
+package org.nuiton.disworkfs;
+
+import org.junit.Before;
+
+public class DisworkFileSystemInMemoryTest extends DisworkFileSystemTest {
+
+ /**
+ * this code executed after {@link DisworkFileSystemTest#setUp()}
+ * @throws Exception
+ */
+ @Before
+ public void setUpFileSystem() throws Exception {
+ // finally, initiate the fileSystem
+ DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig();
+ disworkConfig.setOption("diswork.fs.use_in_memory_map", "true");
+ fileSystem = new DisworkFileSystem(disworkConfig);
+ }
+
+}
Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,82 @@
+package org.nuiton.disworkfs;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.util.ConcurrentModificationException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DisworkFileSystemPastryTest extends DisworkFileSystemTest {
+
+ /**
+ * this code executed after {@link DisworkFileSystemTest#setUp()}
+ * @throws Exception
+ */
+
+ private static final Log log = LogFactory.getLog(DisworkFileSystemPastryTest.class);
+
+ protected Integer bootstrapPort;
+
+ @Before
+ public void setUpFileSystem() throws Exception {
+ // finally, initiate the fileSystem
+ DisworkFileSystemConfig disworkConfig1 = Util.newDisworkConfig();
+ bootstrapPort = disworkConfig1.getUsedPort();
+ fileSystem = new DisworkFileSystem(disworkConfig1);
+
+ }
+
+ /*
+ @Test
+ public void testMultipleNodes1() throws Exception {
+ DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort);
+ DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig);
+
+ assertTrue(fileSystem.exists("/"));
+ assertTrue(fileSystem2.exists("/"));
+ }
+
+ @Test
+ public void testMultipleNodes2() throws Exception {
+ DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort);
+ DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig);
+
+ byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+
+ fileSystem.write("/my_file", new ByteArrayInputStream(bytes));
+
+ assertTrue(fileSystem.exists("/my_file"));
+ assertTrue(fileSystem2.exists("/my_file"));
+
+ assertEquals(1, fileSystem2.readDirectory("/").size());
+
+ byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file"));
+
+ assertArrayEquals(bytes, getResult);
+
+ }
+
+ @Test
+ public void testMultipleNodes3() throws Exception {
+
+ fileSystem.createDirectory("/mydir");
+ try {
+ log.info("----------------------trying to create second directory");
+ fileSystem.createDirectory("/myseconddir");
+ } catch (ConcurrentModificationException e) {
+ fail();
+ }
+
+ }
+ */
+
+}
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -4,6 +4,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -11,6 +12,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ConcurrentModificationException;
import java.util.List;
import java.util.Random;
@@ -22,7 +24,7 @@
import org.nuiton.util.FileUtil;
-public class DisworkFileSystemTest {
+public abstract class DisworkFileSystemTest {
/**
* a place to store files for the test it's a subdirectory of the OS temp
@@ -66,9 +68,6 @@
File randomFile = new File(randomFilePath);
FileUtils.writeByteArrayToFile(randomFile, randomBytes);
- // finally, initiate the fileSystem
- DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig();
- fileSystem = new DisworkFileSystem(disworkConfig);
}
@After
@@ -311,6 +310,18 @@
// ... and only those
assertEquals(3, lsResult.size());
+
+ lsResult = fileSystem.readDirectory("/");
+ assertEquals(1, lsResult.size());
}
-
+
+ @Test
+ public void testConcurrency() throws Exception {
+ fileSystem.createDirectory("/mydir");
+ try {
+ fileSystem.createDirectory("/myseconddir");
+ } catch (ConcurrentModificationException e) {
+ fail();
+ }
+ }
}
Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -1,159 +0,0 @@
-/* *##%
- * Copyright (c) 2010 poussin. All rights reserved.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *##%*/
-
-package org.nuiton.disworkfs;
-
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import org.junit.Test;
-import org.nuiton.disworkfs.storage.EntryUtil;
-
-/**
- *
- * @author poussin
- * @version $Revision$
- *
- * Last update: $Date$
- * by : $Author$
- */
-public class EntryUtilTest {
-
- static protected String directoryEntry =
- EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR
- + "mydir" + EntryUtil.ENTRY_SEPARATOR
- + "myid1";
-
- static protected String fileEntry =
- EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR
- + "myfile" + EntryUtil.ENTRY_SEPARATOR
- + "myid2";
-
- static protected String linkEntry =
- EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR
- + "mylink" + EntryUtil.ENTRY_SEPARATOR
- + "myid3";
-
- static protected String metaBlock =
- "12345" + EntryUtil.BLOCKIDS_SEPARATOR
- + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR
- + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR
- + "myid3";
-
- @Test
- public void testIsDirectory() {
- assertTrue(EntryUtil.isDirectory(directoryEntry));
- assertFalse(EntryUtil.isDirectory(fileEntry));
- }
-
- @Test
- public void testFind() {
-
- String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
- + fileEntry + EntryUtil.ENTRIES_SEPARATOR
- + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
-
- // seek and return entry
- String findResult;
-
- findResult = EntryUtil.findEntryInDirectory(content, "mydir");
- assertEquals(directoryEntry, findResult);
- findResult = EntryUtil.findEntryInDirectory(content, "myfile");
- assertEquals(fileEntry, findResult);
- findResult = EntryUtil.findEntryInDirectory(content, "mylink");
- assertEquals(linkEntry, findResult);
-
-
- // return null if file not found
- findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists");
- assertEquals(null, findResult);
- }
-
- @Test
- public void testGetId() {
- assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry));
- assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry));
- assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry));
- }
-
- @Test
- public void testGetName() {
- assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry));
- assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry));
- assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry));
-
- }
-
- @Test
- public void testGetType() {
- assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry));
- assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry));
- assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry));
- }
-
- @Test
- public void testResolveLink() {
- String path;
-
- path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir");
- assertEquals("/anotherdir", path);
- path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir");
- assertEquals("/dir/subdir/anotherdir", path);
- path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir");
- assertEquals("/dir/anotherdir", path);
- path = EntryUtil.resolveLink("/dir/", "subdir/file");
- assertEquals("/dir/subdir/file", path);
- }
-
- @Test
- public void testBytesToArray() {
- String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$";
- String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s));
- assertEquals(s, copy);
- }
-
- @Test
- public void testGetTotalSizeFromMetaBlock() {
- assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock));
- }
-
- @Test
- public void testGetBlockIdsFromMetaBlock() {
- String[] expectedBlocksIds = {"myid1", "myid2", "myid3"};
- String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock);
- assertArrayEquals(expectedBlocksIds, actualBlocksIds);
- }
-
- @Test
- public void testRemoveEntryFromEntries() {
-
- String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
- + fileEntry + EntryUtil.ENTRIES_SEPARATOR
- + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
-
- String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
- + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
-
- String actual = EntryUtil.removeEntryFromEntries(content, "myfile");
-
- assertEquals(expected, actual);
-
- }
-}
Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-06-02 10:27:22 UTC (rev 54)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -1,27 +0,0 @@
-package org.nuiton.disworkfs;
-
-import static org.junit.Assert.assertFalse;
-
-import java.util.Arrays;
-
-import org.junit.Test;
-import org.nuiton.disworkfs.storage.InMemoryMap;
-
-public class InMemoryMapTest {
-
- /**
- * this test show that the InMemoryMap put implies a copy of the data.
- * After the put, the original value is modified. A final check shows
- * there is no side effect
- */
- @Test
- public void testPut() {
- InMemoryMap map = new InMemoryMap();
- byte[] expected = {0x1, 0x2};
- map.put("key", expected);
- expected[0] = 0xf;
- byte[] actual = map.get("key");
- assertFalse(Arrays.equals(expected, actual));
- }
-
-}
Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,89 @@
+package org.nuiton.disworkfs;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.Socket;
+import java.net.UnknownHostException;
+
+/**
+ * This class provides utilities to write tests easily.
+ *
+ * You can get multiples diswork configs ready to use. All the instances uses
+ * a new port and the local IP.
+ * <pre>
+ * c = newDisworkConfig(); // create a config for a bootstrap node
+ * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that
+ * // will bootstrap by joining the
+ * // first node
+ * c3 = newDisworkConfig(c.getUsedPort())
+ * </pre>
+ *
+ */
+public class Util {
+
+ protected static Integer port = 9000;
+
+ /**
+ * returns a new port, returned value change at each call.
+ * @return
+ */
+ public static Integer getPort() {
+ port += 1;
+ return port;
+ }
+
+ /**
+ * returns the IP on the local machine. Trying to get an public IP or a LAN
+ * IP or the loopback IP if there is no other interface
+ * @return
+ * @throws UnknownHostException
+ */
+ public static InetAddress getIp() throws UnknownHostException {
+ InetAddress result = InetAddress.getLocalHost();
+ if (result.isLoopbackAddress()) {
+ try {
+ Socket temp = new Socket("microsoft.com", 80);
+ result = temp.getLocalAddress();
+ temp.close();
+ } catch (IOException e) {
+ // TODO 20100602 bleny do something ?
+ }
+ }
+ return result;
+ }
+
+ /**
+ * returns a @link {@link DisworkFileSystemConfig} ready to be use as a
+ * config for a single-node instance of DisworkFS
+ * @return
+ * @throws UnknownHostException
+ */
+ public static DisworkFileSystemConfig newDisworkConfig()
+ throws UnknownHostException {
+ return newDisworkConfig(null);
+ }
+
+ /**
+ * returns a @link {@link DisworkFileSystemConfig} ready to be use as a
+ * config for a multiple-node instance of DisworkFS on a same machine.
+ * @param bootstrapPort the port on the same machine where another node can
+ * be found to bootstrap
+ * @return a complete config
+ * @throws UnknownHostException
+ */
+ public static DisworkFileSystemConfig newDisworkConfig(Integer bootstrapPort)
+ throws UnknownHostException {
+ DisworkFileSystemConfig result = new DisworkFileSystemConfig();
+ String port = Util.getPort().toString();
+ String ip = getIp().getHostAddress();
+ result.setOption("diswork.fs.use_in_memory_map", "false");
+ result.setOption("diswork.fs.use_port", port);
+ result.setOption("diswork.fs.bootstrap.ip", ip);
+ if (bootstrapPort == null) {
+ result.setOption("diswork.fs.bootstrap.port", port);
+ } else {
+ result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString());
+ }
+ return result;
+ }
+}
\ No newline at end of file
Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java (from rev 52, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java)
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,159 @@
+/* *##%
+ * Copyright (c) 2010 poussin. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *##%*/
+
+package org.nuiton.disworkfs.storage;
+
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.nuiton.disworkfs.storage.EntryUtil;
+
+/**
+ *
+ * @author poussin
+ * @version $Revision$
+ *
+ * Last update: $Date$
+ * by : $Author$
+ */
+public class EntryUtilTest {
+
+ static protected String directoryEntry =
+ EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR
+ + "mydir" + EntryUtil.ENTRY_SEPARATOR
+ + "myid1";
+
+ static protected String fileEntry =
+ EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR
+ + "myfile" + EntryUtil.ENTRY_SEPARATOR
+ + "myid2";
+
+ static protected String linkEntry =
+ EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR
+ + "mylink" + EntryUtil.ENTRY_SEPARATOR
+ + "myid3";
+
+ static protected String metaBlock =
+ "12345" + EntryUtil.BLOCKIDS_SEPARATOR
+ + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR
+ + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR
+ + "myid3";
+
+ @Test
+ public void testIsDirectory() {
+ assertTrue(EntryUtil.isDirectory(directoryEntry));
+ assertFalse(EntryUtil.isDirectory(fileEntry));
+ }
+
+ @Test
+ public void testFind() {
+
+ String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
+ + fileEntry + EntryUtil.ENTRIES_SEPARATOR
+ + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
+
+ // seek and return entry
+ String findResult;
+
+ findResult = EntryUtil.findEntryInDirectory(content, "mydir");
+ assertEquals(directoryEntry, findResult);
+ findResult = EntryUtil.findEntryInDirectory(content, "myfile");
+ assertEquals(fileEntry, findResult);
+ findResult = EntryUtil.findEntryInDirectory(content, "mylink");
+ assertEquals(linkEntry, findResult);
+
+
+ // return null if file not found
+ findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists");
+ assertEquals(null, findResult);
+ }
+
+ @Test
+ public void testGetId() {
+ assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry));
+ assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry));
+ assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry));
+ }
+
+ @Test
+ public void testGetName() {
+ assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry));
+ assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry));
+ assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry));
+
+ }
+
+ @Test
+ public void testGetType() {
+ assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry));
+ assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry));
+ assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry));
+ }
+
+ @Test
+ public void testResolveLink() {
+ String path;
+
+ path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir");
+ assertEquals("/anotherdir", path);
+ path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir");
+ assertEquals("/dir/subdir/anotherdir", path);
+ path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir");
+ assertEquals("/dir/anotherdir", path);
+ path = EntryUtil.resolveLink("/dir/", "subdir/file");
+ assertEquals("/dir/subdir/file", path);
+ }
+
+ @Test
+ public void testBytesToArray() {
+ String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$";
+ String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s));
+ assertEquals(s, copy);
+ }
+
+ @Test
+ public void testGetTotalSizeFromMetaBlock() {
+ assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock));
+ }
+
+ @Test
+ public void testGetBlockIdsFromMetaBlock() {
+ String[] expectedBlocksIds = {"myid1", "myid2", "myid3"};
+ String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock);
+ assertArrayEquals(expectedBlocksIds, actualBlocksIds);
+ }
+
+ @Test
+ public void testRemoveEntryFromEntries() {
+
+ String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
+ + fileEntry + EntryUtil.ENTRIES_SEPARATOR
+ + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
+
+ String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
+ + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
+
+ String actual = EntryUtil.removeEntryFromEntries(content, "myfile");
+
+ assertEquals(expected, actual);
+
+ }
+}
Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java (from rev 52, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java)
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,27 @@
+package org.nuiton.disworkfs.storage;
+
+import static org.junit.Assert.assertFalse;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.nuiton.disworkfs.storage.InMemoryDisworkMap;
+
+public class InMemoryMapTest {
+
+ /**
+ * this test show that the InMemoryMap put implies a copy of the data.
+ * After the put, the original value is modified. A final check shows
+ * there is no side effect
+ */
+ @Test
+ public void testPut() {
+ InMemoryDisworkMap map = new InMemoryDisworkMap();
+ byte[] expected = {0x1, 0x2};
+ map.put("key", expected);
+ expected[0] = 0xf;
+ byte[] actual = map.get("key");
+ assertFalse(Arrays.equals(expected, actual));
+ }
+
+}
Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-02 12:56:07 UTC (rev 55)
@@ -0,0 +1,98 @@
+package org.nuiton.disworkfs.storage;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.nuiton.disworkfs.DisworkFileSystemConfig;
+import org.nuiton.disworkfs.Util;
+
+public class PastryDisworkMapTest {
+
+ protected static PastryDisworkMap map1;
+ protected static PastryDisworkMap map2;
+
+ protected static byte[] bytes;
+
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ /*
+ if (map1 != null)
+ map1.close();
+ if (map2 != null)
+ map2.close();
+ */
+ }
+
+ @Test
+ public void test() throws Exception {
+ DisworkFileSystemConfig config = Util.newDisworkConfig();
+
+ map1 = new PastryDisworkMap(config);
+
+ byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+ bytes = newBytes;
+
+ map1.put("test", newBytes);
+
+ byte[] getResult = map1.get("test");
+ assertArrayEquals(newBytes, getResult);
+ }
+
+ @Test
+ public void test2nodes() throws Exception {
+ DisworkFileSystemConfig config1 = Util.newDisworkConfig();
+ map1 = new PastryDisworkMap(config1);
+
+ DisworkFileSystemConfig config2 = Util.newDisworkConfig(config1.getUsedPort());
+ map2 = new PastryDisworkMap(config2);
+
+ byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+ bytes = newBytes;
+
+ map1.put("test", newBytes);
+
+ assertTrue(map2.containsKey("test"));
+ assertFalse(map2.containsKey("unknownkey"));
+
+ byte[] getResult = map2.get("test");
+
+ assertArrayEquals(newBytes, getResult);
+
+ byte[] removeResult = map2.remove("test");
+ assertArrayEquals(newBytes, removeResult);
+
+ map1.put("test", newBytes);
+ map2.put("test", newBytes);
+
+ map2.remove("unknow");
+
+ }
+
+ @Test
+ public void testClose() throws Exception {
+ /*
+ DisworkFileSystemConfig config1 = new DisworkFileSystemConfig();
+ map1 = new PastryDisworkMap(config1);
+ */
+ /*
+ try {
+ map2 = new PastryDisworkMap(config1);
+ } catch (IllegalStateException e) {
+ assertEquals("Cannot bind to /192.168.99.119:9001", e.getMessage());
+ }
+ */
+
+ // map1.close();
+ // map2 = new PastryDisworkMap(config1);
+ }
+
+}
1
0
Author: tchemit
Date: 2010-06-02 12:27:22 +0200 (Wed, 02 Jun 2010)
New Revision: 54
Url: http://nuiton.org/repositories/revision/diswork/54
Log:
Utilisation de mavenpom4redmine 2.1.5
Modified:
trunk/pom.xml
Modified: trunk/pom.xml
===================================================================
--- trunk/pom.xml 2010-05-26 08:44:36 UTC (rev 53)
+++ trunk/pom.xml 2010-06-02 10:27:22 UTC (rev 54)
@@ -10,7 +10,7 @@
<parent>
<groupId>org.nuiton</groupId>
<artifactId>mavenpom4redmine</artifactId>
- <version>2.1.4</version>
+ <version>2.1.5</version>
</parent>
<artifactId>diswork</artifactId>
1
0
r53 - in trunk/diswork-fs/src/main/java/org/nuiton/disworkfs: . storage
by bleny＠users.nuiton.org 26 May '10
by bleny＠users.nuiton.org 26 May '10
26 May '10
Author: bleny
Date: 2010-05-26 10:44:36 +0200 (Wed, 26 May 2010)
New Revision: 53
Url: http://nuiton.org/repositories/revision/diswork/53
Log:
exceptions, suppression de donnees retardees
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-25 16:24:08 UTC (rev 52)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-26 08:44:36 UTC (rev 53)
@@ -1,9 +1,11 @@
package org.nuiton.disworkfs;
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.ConcurrentModificationException;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -24,7 +26,7 @@
* on directories, files and symlinks</li>
* </ul>
*/
-public class DisworkFileSystem {
+public class DisworkFileSystem implements Closeable {
/**
* storage will permit to save and read directories, files and links
@@ -104,8 +106,14 @@
* @param path
* @param source
* @throws IOException if file already exists
+ * @throws ConcurrentModificationException if file is already being written
*/
- public void write(String path, InputStream source) throws IOException {
+ public void write(String path, InputStream source)
+ throws IOException,
+ ConcurrentModificationException {
+ if (source == null) {
+ throw new IOException("source is not readable (null stream)");
+ }
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
write(parent, name, source);
@@ -149,13 +157,16 @@
* @param path the absolute path (ending by the name) of the directory
* @throws IOException if parent path is not correct
*/
- public void createDirectory(String path) throws IOException {
+ public void createDirectory(String path)
+ throws IOException,
+ ConcurrentModificationException {
String parent = EntryUtil.getParentFromPath(path);
String dirName = EntryUtil.getNameFromPath(path);
createDirectory(parent, dirName);
}
- protected void createDirectory(String parent, String dirName) throws IOException {
+ protected void createDirectory(String parent, String dirName)
+ throws IOException {
log.info("trying to create directory " + dirName + " in " + parent);
@@ -190,7 +201,8 @@
* @throws IOException if parent path is not correct
*/
public void createSymbolicLink(String path, String target)
- throws IOException {
+ throws IOException,
+ ConcurrentModificationException {
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
createSymbolicLink(parent, name, target);
@@ -244,7 +256,8 @@
* @param path the complete path to the entity to remove
* @throws IOException if path is incorrect or directory not empty
*/
- public void delete(String path) throws IOException {
+ public void delete(String path) throws IOException,
+ ConcurrentModificationException {
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
log.info("trying to remove " + path);
@@ -352,10 +365,9 @@
* @return null if path is not valid or the entry corresponding to path
* @throws IOException
*/
-
- // FIXME 20105021 bleny works fine but is not understandable
protected String walk(String path, String current, String content)
throws IOException {
+ // FIXME 20105021 bleny works fine but is not understandable
String result = null;
if (path.equals("/")) {
@@ -437,5 +449,10 @@
}
return result;
}
+
+ @Override
+ public void close() throws IOException {
+ storage.close();
+ }
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 16:24:08 UTC (rev 52)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-26 08:44:36 UTC (rev 53)
@@ -1,9 +1,12 @@
package org.nuiton.disworkfs.storage;
import java.io.ByteArrayInputStream;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.ConcurrentModificationException;
+import java.util.List;
import java.util.Map;
import org.apache.commons.io.IOUtils;
@@ -14,10 +17,14 @@
/**
*
*/
-public class Storage {
+public class Storage implements Closeable {
private static final Log log = LogFactory.getLog(Storage.class);
+
+ protected static final long LOCK_VALID_TIME = 60 * 60 * 1000;
+ protected List<String> idsToBeRemoved = new ArrayList<String>();
+
/**
* This class is a InputStream used to read data from the map. When
* someone read from the filesytem, he will bre returned a
@@ -147,6 +154,8 @@
}
public void putFile(String id, InputStream content) throws IOException {
+ log.debug("putFile(\"" + id + "\", stream of "+ content.available() +
+ " bytes)");
put(id, content);
}
@@ -171,11 +180,9 @@
* @param value
* @throws IOException
*/
- protected void put(String key, InputStream value) throws IOException {
- // TODO 20100519 bleny deal with null value properly
- if (value == null) {
- value = new ByteArrayInputStream(new byte[0]);
- }
+ protected void put(String key, InputStream value)
+ throws IOException,
+ ConcurrentModificationException {
String lockKey = key + "_lock";
String newDataKey = key + "_newData";
@@ -188,19 +195,20 @@
// file is locked, check date
Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock));
- if (currentDate - currentLock > 60 * 60 * 1000) {
- // this lock is out dated, let's erase it
- String obsoleteMetaBlock = EntryUtil.bytesToString(map.get(newDataKey));
+ if (currentDate - currentLock > LOCK_VALID_TIME) {
+ // this lock is out-dated, let's erase it
+ String obsoleteMetaBlock =
+ EntryUtil.bytesToString(map.get(newDataKey));
if (obsoleteMetaBlock != null) {
- String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock);
+ String[] obsoleteBlocksIds =
+ EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock);
for (String obsoleteBlockId : obsoleteBlocksIds) {
map.remove(obsoleteBlockId);
}
}
} else {
- // file is currently written, stopping
+ // file is currently written, stopping operation
map.put(lockKey, lock);
- // FIXME 20100525 bleny should be declared in throws
throw new ConcurrentModificationException("can't write due to concurrency");
}
}
@@ -253,9 +261,28 @@
map.remove(lockKey);
}
- public void remove(Object key) {
- // TODO poussin 20100518 mark bloque to remove
- map.remove(key);
+ /**
+ * due to concurrency, key should not be removed immediately. Someone may
+ * read it now. It will be removed later, using {@link #clean()}.
+ * @param key the key to be removed from the map
+ */
+ public void remove(String key) {
+ idsToBeRemoved.add(key);
}
+
+ /**
+ * this method clean the map by removing all the blocks that are not
+ * referenced (ie, their key is not in any meta-block or in entries)
+ */
+ public void clean() {
+ for (String key : idsToBeRemoved) {
+ map.remove(key);
+ }
+ }
+ @Override
+ public void close() throws IOException {
+ clean();
+ }
+
}
\ No newline at end of file
1
0
r52 - trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org 25 May '10
by bleny＠users.nuiton.org 25 May '10
25 May '10
Author: bleny
Date: 2010-05-25 18:24:08 +0200 (Tue, 25 May 2010)
New Revision: 52
Url: http://nuiton.org/repositories/revision/diswork/52
Log:
untested implementation of copy-on-write to deal with concurrency
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 15:24:32 UTC (rev 51)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 16:24:08 UTC (rev 52)
@@ -3,6 +3,7 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ConcurrentModificationException;
import java.util.Map;
import org.apache.commons.io.IOUtils;
@@ -165,54 +166,91 @@
}
/**
- * a put in the map, this involves split considerations
+ * a put in the map, this involves split considerations and concurrency
* @param key
* @param value
* @throws IOException
*/
protected void put(String key, InputStream value) throws IOException {
-
- // TODO 20100519 bleny deal with copy-on-write
// TODO 20100519 bleny deal with null value properly
if (value == null) {
value = new ByteArrayInputStream(new byte[0]);
}
+ String lockKey = key + "_lock";
+ String newDataKey = key + "_newData";
+
+ // trying to acquire lock
+ Long currentDate = System.currentTimeMillis();
+ byte[] lock = map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString()));
+
+ if (lock != null) {
+ // file is locked, check date
+ Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock));
+
+ if (currentDate - currentLock > 60 * 60 * 1000) {
+ // this lock is out dated, let's erase it
+ String obsoleteMetaBlock = EntryUtil.bytesToString(map.get(newDataKey));
+ if (obsoleteMetaBlock != null) {
+ String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock);
+ for (String obsoleteBlockId : obsoleteBlocksIds) {
+ map.remove(obsoleteBlockId);
+ }
+ }
+ } else {
+ // file is currently written, stopping
+ map.put(lockKey, lock);
+ // FIXME 20100525 bleny should be declared in throws
+ throw new ConcurrentModificationException("can't write due to concurrency");
+ }
+ }
+
+ // here, we know we can write or an exception
+ // would have been thrown earlier
String blocksIds = "";
int readResult = 0;
int totalSize = 0;
+ String metaBlock = totalSize + blocksIds;
+ map.put(newDataKey, EntryUtil.stringToBytes(metaBlock));
+
// creating a buffer of the size of a block
int bufferSize = disworkConfig.getBlockSize();
byte[] buffer = new byte[bufferSize];
-
+
while ((readResult = value.read(buffer)) != -1) {
totalSize += readResult;
-
+
byte[] newBlock = buffer;
-
+
// if the buffer is not full, truncate the block
if (readResult < buffer.length) {
newBlock = new byte[readResult];
System.arraycopy(buffer, 0, newBlock, 0, readResult);
}
-
+
String id = EntryUtil.generateId();
blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id;
log.debug("saving new block (size = " + newBlock.length + ")"
+ " at key " + id);
-
+
// copy block in map
map.put(id, newBlock);
+
+ metaBlock = totalSize + blocksIds;
+
+ log.debug("putting metablock " + metaBlock + " at key " + key);
+ map.put(newDataKey, EntryUtil.stringToBytes(metaBlock));
+
+ // updating lock
+ currentDate = System.currentTimeMillis();
+ map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString()));
}
- String metaBlock = totalSize + blocksIds;
-
- log.debug("putting metablock " + metaBlock + " at key " + key);
-
- map.put(key, EntryUtil.stringToBytes(metaBlock));
+ map.put(key, map.get(newDataKey));
+ map.remove(lockKey);
}
public void remove(Object key) {
1
0
r51 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage main/resources test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 25 May '10
by bleny＠users.nuiton.org 25 May '10
25 May '10
Author: bleny
Date: 2010-05-25 17:24:32 +0200 (Tue, 25 May 2010)
New Revision: 51
Url: http://nuiton.org/repositories/revision/diswork/51
Log:
menage, documentation, tests
Added:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java
Removed:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
trunk/diswork-fs/src/main/resources/log4j.properties
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java
Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -1,28 +0,0 @@
-package org.nuiton.disworkfs;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-
-public interface DistributedFileSystem {
-
- public boolean exists(String path) throws IOException;
-
- public void write(String path, InputStream source) throws IOException;
-
- public void write(String parent, String fileName, InputStream source)
- throws IOException;
-
- public InputStream read(String path) throws FileNotFoundException,
- IOException;
-
- public void mkdir(String path) throws IOException;
-
- public void ln(String path, String target) throws IOException;
-
- public void remove(String path) throws IOException;
-
- public List<String> ls(String path) throws IOException;
-
-}
Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -1,14 +0,0 @@
-package org.nuiton.disworkfs;
-
-import org.nuiton.util.ApplicationConfig;
-
-public class DisworkConfig extends ApplicationConfig {
-
- public DisworkConfig() {
- setDefaultOption("blocks_size", "10485760"); // 10 MiB
- }
-
- public int getBlockSize() {
- return Integer.parseInt(getOption("blocks_size"));
- }
-}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -12,122 +12,67 @@
import org.nuiton.disworkfs.storage.InMemoryMap;
import org.nuiton.disworkfs.storage.Storage;
-public class DisworkFileSystem implements DistributedFileSystem {
+/** Main class of Diswork File System, provide methods for all operations
+ * You can use:
+ * <ul>
+ * <li>{@link #createDirectory(String)} and {@link DisworkFileSystem#readDirectory(String)}
+ * to create and browse directories</li>
+ * <li>{@link #write(String, InputStream)} and {@link #read(String)}
+ * to write a file and read it</li>
+ * <li>{@link #createSymbolicLink(String, String)} to create symbolic links</li>
+ * <li>{@link #exists(String)} and {@link #delete(String)} can be used
+ * on directories, files and symlinks</li>
+ * </ul>
+ */
+public class DisworkFileSystem {
+ /**
+ * storage will permit to save and read directories, files and links
+ */
protected Storage storage;
private static final Log log = LogFactory.getLog(DisworkFileSystem.class);
- public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException {
+ /** default constructor (uses default configuration parameters)
+ * @throws IOException
+ */
+ public DisworkFileSystem() throws IOException {
+ this(new DisworkFileSystemConfig());
+ }
+
+ /** constructor allowing to provide another configuration to use
+ *
+ * @param disworkConfig the configuration
+ * @throws IOException
+ */
+ public DisworkFileSystem(DisworkFileSystemConfig disworkConfig)
+ throws IOException {
storage = new Storage(disworkConfig, new InMemoryMap());
storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent());
}
- @Override
+ /** tests the existence of a file/dir/link at a given path
+ * return true if something exists at path p, it will be true if a call
+ * to <code>mkdir(p)</code>, <code>write(p, ?)</code> or
+ * <code>ln(p, ?)</code> has been done before.
+ * @param path a path in the virtual FS
+ * @return true is something (a link, a file, or a directory) exists at path
+ * @throws IOException
+ */
public boolean exists(String path) throws IOException {
String entry = walk(path);
boolean result = entry != null;
return result;
}
-
+
/**
- * return the entry of the element at the end of <code>path</code>
- *
- * @param path
- * @return null if path is not valid
+ * use this method to read the content of a file. A call of read may be
+ * done after a call to {@link #write}
+ * @param path the path to the file to read
+ * @return an InputStream on the file
+ * @throws FileNotFoundException if no file exists at this path
+ * @throws IOException if path exists but is a directory
*/
- protected String walk(String path) throws IOException {
- String result = walk(path, null, null);
- log.info("walking to " + path + " returns " + result);
- return result;
- }
-
- // FIXME 20105021 bleny works fine but is not understandable
- protected String walk(String path, String current, String content)
- throws IOException {
- String result = null;
-
- if (path.equals("/")) {
- return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR
- + "/" + EntryUtil.ENTRY_SEPARATOR
- + "/";
- }
-
-
- String parentPath = EntryUtil.getParentFromPath(path);
-
- if (content == null) {
- // start of walk
- content = storage.getRootDirectory();
- result = walk(path, "/", content);
- } else if (parentPath.equals(current)) {
- // in the last directory
- // recuperation de l'element a traiter dans p
-
- String tail = path.substring(current.length());
-
- /*
- log.info("tail " + tail);
- String[] elementsNames = tail.split("/");
- String p = elementsNames[0];
- */
- String p = EntryUtil.getNameFromPath(tail);
-
- log.info("in final dir " + current + ", looking for " + p);
-
- String entry = EntryUtil.findEntryInDirectory(content, p);
- result = entry;
- } else {
- // in middle of path
-
- String tail; // the path still remaining when in current
-
- if (current.equals("/")) {
- tail = path.substring(current.length());
- } else {
- tail = path.substring(current.length() + 1);
- }
-
- log.debug("current = " + current);
- log.debug("tail = " + tail);
- String[] elementsNames = tail.split("/");
- String p = elementsNames[0];
-
- log.info("in intermediate dir " + current + ", looking for " + p);
-
- // mise a jour de current
- if (current.equals("/")) current = "";
- current += "/" + p;
-
- String entry = EntryUtil.findEntryInDirectory(content, p);
- if (entry == null) {
- result = null;
- } else {
- if (EntryUtil.isDirectory(entry)) {
- String id = EntryUtil.getId(entry);
- content = storage.getDirectory(id);
- result = walk(path, current, content);
- } else if (EntryUtil.isLink(entry)) {
- String id = EntryUtil.getId(entry);
- String linkContent = storage.getLink(id);
- String newTarget = EntryUtil.resolveLink(current, linkContent);
- newTarget += path.substring(current.length());
-
- // restart walk from /
- result = walk(newTarget, null, null);
- } else if (EntryUtil.isFile(entry)) {
- // erreur, find file in path like '/dir1/dir2/filename/dir3'
- result = null;
- } else {
- log.warn("strange case : " + entry);
- result = null;
- }
- }
- }
- return result;
- }
-
- @Override
public InputStream read(String path) throws FileNotFoundException,
IOException {
@@ -140,7 +85,7 @@
if (EntryUtil.isLink(entry)) {
log.info("reading link " + path);
- String id = EntryUtil.getId(entry);
+ String id = EntryUtil.getIdFromEntry(entry);
String link = storage.getLink(id);
String newTarget = EntryUtil.resolveLink(path, link);
result = read(newTarget);
@@ -148,21 +93,26 @@
throw new IOException("target is not a file: " + path);
} else if (EntryUtil.isFile(entry)) {
log.info("reading file " + path);
- String id = EntryUtil.getId(entry);
+ String id = EntryUtil.getIdFromEntry(entry);
result = storage.getFile(id);
- }
+ }
return result;
}
- @Override
+ /**
+ * write a file on the file system at a given place
+ * @param path
+ * @param source
+ * @throws IOException if file already exists
+ */
public void write(String path, InputStream source) throws IOException {
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
write(parent, name, source);
}
-
- @Override
- public void write(String parent, String fileName, InputStream source) throws IOException {
+
+ protected void write(String parent, String fileName, InputStream source)
+ throws IOException {
String entryParent = walk(parent);
if (entryParent == null) {
@@ -170,11 +120,20 @@
}
if (EntryUtil.isDirectory(entryParent)) {
- String parentId = EntryUtil.getId(entryParent);
+ String parentId = EntryUtil.getIdFromEntry(entryParent);
String content = storage.getDirectory(parentId);
+
+ // checking that file do not already exists in this directory
+ String findResult = EntryUtil.findEntryInDirectory
+ (content, fileName);
+ if (findResult != null) {
+ throw new IOException
+ (parent + " already contains an element named " + fileName);
+ }
+
String newFileId = EntryUtil.generateId();
String newContent = EntryUtil.addEntryToDirectoryContent(
- content, EntryUtil.TYPE.F, fileName, newFileId);
+ content, EntryUtil.TYPE.F, fileName, newFileId);
// store file before meta info
storage.putFile(newFileId, source);
@@ -185,14 +144,18 @@
}
}
- @Override
- public void mkdir(String path) throws IOException {
+ /**
+ * creates a new empty directory
+ * @param path the absolute path (ending by the name) of the directory
+ * @throws IOException if parent path is not correct
+ */
+ public void createDirectory(String path) throws IOException {
String parent = EntryUtil.getParentFromPath(path);
String dirName = EntryUtil.getNameFromPath(path);
- mkdir(parent, dirName);
+ createDirectory(parent, dirName);
}
- public void mkdir(String parent, String dirName) throws IOException {
+ protected void createDirectory(String parent, String dirName) throws IOException {
log.info("trying to create directory " + dirName + " in " + parent);
@@ -203,7 +166,7 @@
}
if (EntryUtil.isDirectory(entryParent)) {
- String parentId = EntryUtil.getId(entryParent);
+ String parentId = EntryUtil.getIdFromEntry(entryParent);
String content = storage.getDirectory(parentId);
String newDirectoryId = EntryUtil.generateId();
String newContent = EntryUtil.addEntryToDirectoryContent(
@@ -219,14 +182,25 @@
}
}
- @Override
- public void ln(String path, String target) throws IOException {
+ /**
+ * create a new symbolic link
+ * @param path the path (ending by the name) where the link will be created
+ * @param target the path where the link point to
+ * (may be relative or absolute)
+ * @throws IOException if parent path is not correct
+ */
+ public void createSymbolicLink(String path, String target)
+ throws IOException {
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
- ln(parent, name, target);
+ createSymbolicLink(parent, name, target);
}
- public void ln(String parent, String name, String target) throws IOException {
+ /**
+ * @see #createSymbolicLink(String, String)
+ */
+ protected void createSymbolicLink(String parent, String name, String target)
+ throws IOException {
if (target.startsWith("/")) {
// target is absolute
@@ -244,11 +218,11 @@
}
if (EntryUtil.isDirectory(entryParent)) {
- String parentId = EntryUtil.getId(entryParent);
+ String parentId = EntryUtil.getIdFromEntry(entryParent);
String content = storage.getDirectory(parentId);
String newLinkId = EntryUtil.generateId();
String newContent = EntryUtil.addEntryToDirectoryContent(
- content, EntryUtil.TYPE.L, name, newLinkId);
+ content, EntryUtil.TYPE.L, name, newLinkId);
// store link before meta info
storage.putLink(newLinkId, target);
@@ -264,15 +238,23 @@
}
}
- @Override
- public void remove(String path) throws IOException {
+ /**
+ * remove a file, directory, or link. Non-empty directories can't be
+ * removed
+ * @param path the complete path to the entity to remove
+ * @throws IOException if path is incorrect or directory not empty
+ */
+ public void delete(String path) throws IOException {
String parent = EntryUtil.getParentFromPath(path);
String name = EntryUtil.getNameFromPath(path);
log.info("trying to remove " + path);
- remove(parent, name);
+ delete(parent, name);
}
-
- protected void remove(String parent, String name) throws IOException {
+
+ /**
+ * @see #delete(String)
+ */
+ protected void delete(String parent, String name) throws IOException {
String entryParent = walk(parent);
if (entryParent == null) {
@@ -280,25 +262,28 @@
}
if (EntryUtil.isDirectory(entryParent)) {
- String parentId = EntryUtil.getId(entryParent);
+ String parentId = EntryUtil.getIdFromEntry(entryParent);
String content = storage.getDirectory(parentId);
String entry = EntryUtil.findEntryInDirectory(content, name);
- String idToRemove = EntryUtil.getId(entry);
+ String idToRemove = EntryUtil.getIdFromEntry(entry);
String newContent = EntryUtil.removeEntryFromEntries(
content, name);
// check if not removing a non-empty directory
if (EntryUtil.isDirectory(entry)) {
- String innerDirectoryId = EntryUtil.getId(entry);
- String innerDirectoryContent = storage.getDirectory(innerDirectoryId);
- if (! innerDirectoryContent.equals(EntryUtil.newEmptyDirectoryContent())) {
+ String innerDirectoryId = EntryUtil.getIdFromEntry(entry);
+ String innerDirectoryContent =
+ storage.getDirectory(innerDirectoryId);
+ // checking the emptiness of the directory
+ if (!innerDirectoryContent.equals(
+ EntryUtil.newEmptyDirectoryContent())) {
// directory is not empty
- throw new IOException("trying to remove a non-empty directory");
+ throw new IOException
+ ("trying to remove a non-empty directory");
}
}
-
// update meta info directory
storage.putDirectory(parentId, newContent);
@@ -309,27 +294,34 @@
throw new IOException(parent + " is not a directory");
}
}
-
- @Override
- public List<String> ls(String path) throws IOException {
+
+ /**
+ * list the content of a directory
+ * @param path the complete path to the directory
+ * @return a list of the names of the elements in <code>path</code>
+ * @throws IOException if path doesn't point to a directory
+ */
+ public List<String> readDirectory(String path) throws IOException {
String entry = walk(path);
List<String> result = null;
-
if (entry == null) {
throw new IOException(path + " doesn't exists");
} else {
+ // path may be a link, if it's the case,
+ // entry become the actual directory
if (EntryUtil.isLink(entry)) {
- String target = storage.getLink(EntryUtil.getId(entry));
+ String target = storage.getLink(
+ EntryUtil.getIdFromEntry(entry));
entry = walk(target);
- }
-
+ }
if (EntryUtil.isDirectory(entry)) {
- String content = storage.getDirectory(EntryUtil.getId(entry));
+ String content = storage.getDirectory(
+ EntryUtil.getIdFromEntry(entry));
String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR);
result = new ArrayList<String>();
for (String elementEntry : entries) {
- result.add(EntryUtil.getName(elementEntry));
+ result.add(EntryUtil.getNameFromEntry(elementEntry));
}
} else {
throw new IOException(path + " is not a directory");
@@ -339,4 +331,111 @@
return result;
}
+ /**
+ * return the entry of the element at the end of <code>path</code>
+ * @param path
+ * @return null if path is not valid
+ */
+ protected String walk(String path) throws IOException {
+ String result = walk(path, null, null);
+ log.info("walking to " + path + " returns " + result);
+ return result;
+ }
+
+ /**
+ * This method is a recursive function to walk through the tree structure
+ * of the directories and starting for root directory, following the
+ * symbolic links to reach the given path
+ * @param path
+ * @param current
+ * @param content
+ * @return null if path is not valid or the entry corresponding to path
+ * @throws IOException
+ */
+
+ // FIXME 20105021 bleny works fine but is not understandable
+ protected String walk(String path, String current, String content)
+ throws IOException {
+ String result = null;
+
+ if (path.equals("/")) {
+ return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR
+ + "/" + EntryUtil.ENTRY_SEPARATOR
+ + "/";
+ }
+
+ String parentPath = EntryUtil.getParentFromPath(path);
+
+ if (content == null) {
+ // start of walk
+ content = storage.getRootDirectory();
+ result = walk(path, "/", content);
+ } else if (parentPath.equals(current)) {
+ // in the last directory
+ // recuperation de l'element a traiter dans p
+
+ String tail = path.substring(current.length());
+
+ /*
+ log.info("tail " + tail);
+ String[] elementsNames = tail.split("/");
+ String p = elementsNames[0];
+ */
+ String p = EntryUtil.getNameFromPath(tail);
+
+ log.info("in final dir " + current + ", looking for " + p);
+
+ String entry = EntryUtil.findEntryInDirectory(content, p);
+ result = entry;
+ } else {
+ // in middle of path
+
+ String tail; // the path still remaining when in current
+
+ if (current.equals("/")) {
+ tail = path.substring(current.length());
+ } else {
+ tail = path.substring(current.length() + 1);
+ }
+
+ log.debug("current = " + current);
+ log.debug("tail = " + tail);
+ String[] elementsNames = tail.split("/");
+ String p = elementsNames[0];
+
+ log.info("in intermediate dir " + current + ", looking for " + p);
+
+ // mise a jour de current
+ if (current.equals("/")) current = "";
+ current += "/" + p;
+
+ String entry = EntryUtil.findEntryInDirectory(content, p);
+ if (entry == null) {
+ result = null;
+ } else {
+ if (EntryUtil.isDirectory(entry)) {
+ String id = EntryUtil.getIdFromEntry(entry);
+ content = storage.getDirectory(id);
+ result = walk(path, current, content);
+ } else if (EntryUtil.isLink(entry)) {
+ String id = EntryUtil.getIdFromEntry(entry);
+ String linkContent = storage.getLink(id);
+ String newTarget =
+ EntryUtil.resolveLink(current, linkContent);
+ newTarget += path.substring(current.length());
+
+ // restart walk from /
+ result = walk(newTarget, null, null);
+ } else if (EntryUtil.isFile(entry)) {
+ // error, found file in path like '/dir1/dir2/filename/dir3'
+ result = null;
+ } else {
+ log.warn("strange case: " + entry);
+ result = null;
+ }
+ }
+ }
+ return result;
+ }
+
}
Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java (from rev 50, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java)
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -0,0 +1,26 @@
+package org.nuiton.disworkfs;
+
+import org.nuiton.util.ApplicationConfig;
+
+/**
+ * This object contains parameters used in {@link DisworkFileSystem}.
+ *
+ * Available parameters are:
+ * <dl>
+ * <dt>blocks_size</dt>
+ * <dd>
+ * The size of blocks to create when splitting big data in bytes
+ * (by default, 10 MiB)
+ * </dd>
+ * </dl>
+ */
+public class DisworkFileSystemConfig extends ApplicationConfig {
+
+ public DisworkFileSystemConfig() {
+ setDefaultOption("blocks_size", "10485760"); // 10 MiB
+ }
+
+ public int getBlockSize() {
+ return getOptionAsInt("blocks_size");
+ }
+}
Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -0,0 +1,10 @@
+/**
+ * DisworkFS is a distributed file system. You can use it by instantiating
+ * {@link DisworkFileSystem} and use it to store directories and their content.
+ *
+ * You can change the default DisworkFileSystem behavior by provide a
+ * {@link DisworkFileSystemConfig} instance at construction. If you don't
+ * provide one, a instance will be created
+ */
+
+package org.nuiton.disworkfs;
\ No newline at end of file
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -7,28 +7,51 @@
import org.apache.commons.io.FilenameUtils;
/**
+ * This class contains String utilities to ease operations on
+ * <dl>
+ * <dt>a single entry;</dt>
+ * <dd>a record in the list of content in a directory</dd>
+ * <dt>multiple entries;</dt>
+ * <dd>a string containing multiple entry (as above) separated by
+ * {@link #ENTRIES_SEPARATOR}
+ * </dd>
+ * <dt>a metablock;</dt>
+ * <dd>a metablock is a String that begins with the total size of the data
+ * it describes and the ids where some blocks containing the data
+ * are stored into the map</dd>
+ * <dt>a path.</dt>
+ * <dd>a string like /a/b/c/d</dd>
+ * </dl>
+ *
* @author poussin
* @version $Revision$
*
* Last update: $Date$
- * by : $Author$
+ * by: $Author$
*/
public class EntryUtil {
+ static public enum TYPE { /** Directory */ D,
+ /** Link */ L,
+ /** File */ F
+ };
+ static final int TYPE_LENGTH = 1;
+
static final public String ENTRY_SEPARATOR = ":";
static final public String ENTRIES_SEPARATOR = "\n";
static final public String BLOCKIDS_SEPARATOR = ";";
-
- static final int TYPE_LENGTH = 1;
- /*
- // this line may not be accurate for "/"
- static final int UUID_LENGTH = UUID.randomUUID().toString().length();
- */
+
static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length();
- static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/};
+ /**
+ * This charset is forced in string-to-bytes and bytes-to-string conversions
+ */
+ static protected Charset CHARSET = Charset.forName("UTF-8");
- static protected Charset CHARSET = Charset.forName("UTF-8");
+ /**
+ * This is a utility class thus should not be instantiated
+ */
+ protected EntryUtil() {};
/**
* generate a new Id usage as a new key for the map
@@ -45,7 +68,7 @@
* a link
*/
static public boolean isDirectory(String entry) {
- boolean result = entry.startsWith(TYPE.D.name());
+ boolean result = getTypeFromEntry(entry) == TYPE.D;
return result;
}
@@ -55,7 +78,7 @@
* a directory
*/
static public boolean isLink(String entry) {
- boolean result = entry.startsWith(TYPE.L.name());
+ boolean result = getTypeFromEntry(entry) == TYPE.L;
return result;
}
@@ -65,47 +88,31 @@
* a directory
*/
static public boolean isFile(String entry) {
- boolean result = entry.startsWith(TYPE.F.name());
+ boolean result = getTypeFromEntry(entry) == TYPE.F;
return result;
}
- static public TYPE getType(String entry) {
+ static public TYPE getTypeFromEntry(String entry) {
int index = entry.indexOf(ENTRY_SEPARATOR);
String type = entry.substring(0, index);
TYPE result = TYPE.valueOf(type);
return result;
}
- static public String getName(String entry) {
+ static public String getNameFromEntry(String entry) {
int start = entry.indexOf(ENTRY_SEPARATOR);
int last = entry.lastIndexOf(ENTRY_SEPARATOR);
String result = entry.substring(start + ENTRY_SEPARATOR_LENGTH, last);
return result;
}
- static public String getId(String entry) {
+ static public String getIdFromEntry(String entry) {
int index = entry.lastIndexOf(ENTRY_SEPARATOR);
String result = entry.substring(index + ENTRY_SEPARATOR_LENGTH);
return result;
}
+
- static public int getTotalSizeFromMetaBlock(String metaBlock) {
- // dealing with empty meta-block (for empty directory or file)
- if (metaBlock.equals("0")) return 0;
- String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR));
- return Integer.valueOf(result);
- }
-
- static public String[] getBlockIdsFromMetaBlock(String metaBlock) {
- // dealing with empty meta-block (for empty directory or file)
- if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) {
- return new String[0];
- }
- String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length());
- String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR);
- return blockIds;
- }
-
/**
* find file name in directory content description
*
@@ -126,6 +133,12 @@
return result;
}
+ public static String removeEntryFromEntries(String content, String name) {
+ String entry = findEntryInDirectory(content, name);
+ String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, "");
+ return newContent;
+ }
+
/**
*
*/
@@ -135,51 +148,38 @@
/**
*
- * @param content the string content of the directory (all entries before add)
+ * @param content the string content of the directory
+ * (all entries before add)
* @param type the type of the new entry
* @param name the name (not a full path) of the new entry
* @param id the id where the content of the entry can be found
* @return a String with the new content
*/
static public String addEntryToDirectoryContent
- (String content, TYPE type, String name, String id) {
- content += type.name() + ENTRY_SEPARATOR + name +
- ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR;
- return content;
+ (String content, TYPE type, String name, String id) {
+ String newEntry = type.name() + ENTRY_SEPARATOR + name +
+ ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR;
+ return content + newEntry;
}
- /**
- * Resolve a path from a parent directory,
- *
- * ie resolve /dir/subdir, ../file returns /dir/file
- * @param parent the path to the dir where start from compute the path
- * @param link the path to follow (may be absolute or relative to
- * <code>parent</code>)
- * @return the path to the destination
- */
- static public String resolveLink(String parent, String link) {
- /*
- String result = null;
- if (link.startsWith("/")) {
- // destination is absolute, ignore parent
- result = link;
- } else {
- // destination is relative, so we have to consider parent
- // parent may have a trailing file name, let's remove it
- String parentWithNoFile = parent;
- if (!parent.endsWith("/")) {
- // removing the file at the end of the path, it's not a
- // directory and thus should not be part of the path
- parentWithNoFile = parent.substring(0, parent.lastIndexOf("/") + 1);
- }
-
- // concatenate and making the complete path consistent
- // ie /dir/subdir/../file to /dir/file
- result = FilenameUtils.normalize(parentWithNoFile + link);
+ static public int getTotalSizeFromMetaBlock(String metaBlock) {
+ // dealing with empty meta-block (for empty directory or file)
+ if (metaBlock.equals("0")) return 0;
+ String result = metaBlock.substring(0,
+ metaBlock.indexOf(BLOCKIDS_SEPARATOR));
+ return Integer.valueOf(result);
+ }
+
+ static public String[] getBlockIdsFromMetaBlock(String metaBlock) {
+ // dealing with empty meta-block (for empty directory or file)
+ if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) {
+ return new String[0];
}
- return result;
- */
- return FilenameUtils.concat(parent, link);
+ String blockIdsAsString = metaBlock.substring(
+ metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1,
+ metaBlock.length());
+ String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR);
+ return blockIds;
}
/**
@@ -189,20 +189,31 @@
* @return a byte array containing all the bytes in <code>string</code>
*/
public static byte[] stringToBytes(String string) {
- // FIXME 20100519 bleny this code may cause bad conversion
- // due to charsets
byte[] bytes = string.getBytes(CHARSET);
return bytes;
}
/**
- * @see {@link #stringToBytes(String)}
- * @param string the string
+ * @see #stringToBytes(String)
+ * @param bytes the bytes to convert
* @return a byte array containing all the bytes in <code>string</code>
*/
public static String bytesToString(byte[] bytes) {
return new String(bytes, CHARSET);
}
+
+ /**
+ * Resolve a path from a parent directory and a link
+ *
+ * ie resolve /dir/subdir, ../file returns /dir/file
+ * @param parent the path to the dir where start from compute the path
+ * @param link the path to follow (may be absolute or relative to
+ * <code>parent</code>)
+ * @return the path to the destination
+ */
+ static public String resolveLink(String parent, String link) {
+ return FilenameUtils.concat(parent, link);
+ }
public static String getParentFromPath(String path) {
String parent = FilenameUtils.getFullPathNoEndSeparator(path);
@@ -215,11 +226,5 @@
public static String getNameFromPath(String path) {
return FilenameUtils.getName(path);
}
-
- public static String removeEntryFromEntries(String content, String name) {
- String entry = findEntryInDirectory(content, name);
- String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, "");
- return newContent;
- }
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -6,6 +6,11 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+/** An in-memory implementation of a Map
+ * This class is a map where put make a copy of the value. It is used for
+ * stubbing purpose. It emulates the behavior of a DHT and thus can be used
+ * by {@link Storage} to store data and blocks.
+ */
public class InMemoryMap extends HashMap<String, byte[]> {
private final Log log = LogFactory.getLog(InMemoryMap.class);
@@ -17,4 +22,4 @@
log.info("putting " + valueCopy.length + " bytes");
return super.put(key, valueCopy);
}
-}
+}
\ No newline at end of file
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -8,12 +8,20 @@
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.nuiton.disworkfs.DisworkConfig;
+import org.nuiton.disworkfs.DisworkFileSystemConfig;
+/**
+ *
+ */
public class Storage {
private static final Log log = LogFactory.getLog(Storage.class);
+ /**
+ * This class is a InputStream used to read data from the map. When
+ * someone read from the filesytem, he will bre returned a
+ * SplitBlocksInputStream. This InputStream loads blocks only when needed
+ */
protected class SplitBlocksInputStream extends InputStream {
protected Map<String, byte[]> map;
@@ -45,6 +53,8 @@
if (currentBlock == null || currentBlock.available() == 0) {
blockIdsIndex += 1;
if (blockIdsIndex < blockIds.length) {
+ // closing previously opened stream
+ IOUtils.closeQuietly(currentBlock);
currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex]));
log.debug("new current block (size = " + currentBlock.available() + ")");
} else {
@@ -53,16 +63,18 @@
}
int result = currentBlock.read();
- /*
- log.debug("reading block number " + blockIdsIndex +
- " (available = " + available + ")" +
- " returns " + result);
- */
available -= 1;
-
+
+ if(log.isDebugEnabled()) {
+ log.debug("reading block number " + blockIdsIndex +
+ " (available = " + available + ")" +
+ " returns " + result);
+ }
return result;
}
+ // This method is called at the first read or the first available
+ // it initialize all the necessary field
protected void checkInitialization() {
if (blockIds == null) {
byte[] bytes = map.get(id);
@@ -86,17 +98,13 @@
protected Map<String, byte[]> map;
- protected DisworkConfig disworkConfig;
+ protected DisworkFileSystemConfig disworkConfig;
- public Storage(DisworkConfig disworkConfig, Map<String, byte[]> map) {
+ public Storage(DisworkFileSystemConfig disworkConfig, Map<String, byte[]> map) {
this.map = map;
this.disworkConfig = disworkConfig;
}
- public boolean contains(Object id) {
- return map.containsKey(id);
- }
-
/**
* @return the content (entries) of the root directory
* @throws IOException
@@ -108,8 +116,11 @@
-
-
+ /**
+ * @param id an id of a directory
+ * @return a the entries of the directory
+ * @throws IOException
+ */
public String getDirectory(String id) throws IOException {
InputStream in = get(id);
String content = IOUtils.toString(in);
@@ -162,36 +173,37 @@
protected void put(String key, InputStream value) throws IOException {
// TODO 20100519 bleny deal with copy-on-write
- // TODO 20100519 bleny deal with null value properly
+ // TODO 20100519 bleny deal with null value properly
if (value == null) {
value = new ByteArrayInputStream(new byte[0]);
}
String blocksIds = "";
- int read = 0;
+ int readResult = 0;
int totalSize = 0;
+ // creating a buffer of the size of a block
int bufferSize = disworkConfig.getBlockSize();
byte[] buffer = new byte[bufferSize];
- while ((read = value.read(buffer)) != -1) {
- totalSize += read;
+ while ((readResult = value.read(buffer)) != -1) {
+ totalSize += readResult;
byte[] newBlock = buffer;
- // if the block is shorter, truncate
- if (read < buffer.length) {
- newBlock = new byte[read];
- System.arraycopy(buffer, 0, newBlock, 0, read);
+ // if the buffer is not full, truncate the block
+ if (readResult < buffer.length) {
+ newBlock = new byte[readResult];
+ System.arraycopy(buffer, 0, newBlock, 0, readResult);
}
String id = EntryUtil.generateId();
blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id;
- log.debug("saving new block (size = " + newBlock.length
- + ") at key " + id);
+ log.debug("saving new block (size = " + newBlock.length + ")"
+ + " at key " + id);
- // copy buffer in map
+ // copy block in map
map.put(id, newBlock);
}
@@ -208,5 +220,4 @@
map.remove(key);
}
-
-}
+}
\ No newline at end of file
Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -0,0 +1,203 @@
+/**
+ * <p>
+ * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a
+ * way to persistently store data. This is done by the {@link Storage} class
+ * which permit to store different type of data.
+ * </p>
+ * <p>
+ * We illustrate the use of a file system by setting the content of "/",
+ * the root directory as follow:
+ * </p>
+ * <ul>
+ * <li>
+ * a directory named "my_folder" that contains:
+ * <ul>
+ * <li>
+ * a directory named "my_sub_folder" that contains
+ * <ul>
+ * <li>a file name "my_deep_file.ext"</li>
+ * <li>an empty folder named "my_empty_folder"</li>
+ * </ul>
+ * </li>
+ * <li>a file named "my_other_file.ext"</li>
+ * </ul>
+ * </li>
+ * <li>
+ * a file named "my_file.ext" (containing binary data: 010101)
+ * </li>
+ * <li>
+ * a symbolic link named "my_link" which target the path
+ * <code>"/my_folder/my_sub_folder/my_deep_file.ext"</code>
+ * </li>
+ * </ul>
+ *
+ * Storage uses a Map to store data.
+ *
+ * <table>
+ * <caption>Use of map, abstract point of view</caption>
+ * <tr>
+ * <th>Key (String)</th>
+ * <th>Value (byte[])</th>
+ * </tr>
+ * <tr>
+ * <td>"/"</td>
+ * <td>"D:my_folder:ID1\n<br />
+ * F:my_file.ext:ID2\n<br />
+ * L:my_link:ID3"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID1"</td>
+ * <td>"D:my_sub_folder:ID4\n<br />
+ * F:my_other_file.ext:ID5"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID2"</td>
+ * <td>010101</td>
+ * </tr>
+ * <tr>
+ * <td>"ID3"</td>
+ * <td>"/my_folder/my_sub_folder/my_deep_file.ext"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID4"</td>
+ * <td>"F:my_deep_file:ID6\n<br />
+ * D:my_empty_folder:ID7"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID5"</td>
+ * <td>011010100111010...</td>
+ * </tr>
+ * <tr>
+ * <td>"ID6"</td>
+ * <td>1100010101010110110010...</td>
+ * </tr>
+ * <tr>
+ * <td>"ID7"</td>
+ * <td>""</td>
+ * </tr>
+ * </table>
+ *
+ * <p>
+ * An entry is, for example, "D:my_folder:ID1". It contains the type of the
+ * entry (<strong>D</strong>irectory, <strong>F</strong>ile or
+ * <strong>L</strong>ink), the name of the element, and an ID to be used as
+ * a key on the map to get the actual content. Those three informations are
+ * separated by ":" which is {@link EntryUtil#ENTRY_SEPARATOR}.
+ * </p>
+ *
+ * <p>
+ * A directory way have multiple <em>entries</em>. Entries of a directory are
+ * separated by "\n" which is {@link EntryUtil#ENTRIES_SEPARATOR}.
+ * </p>
+ * <p>
+ * The above description shows the main principle used to store a tree
+ * structure in a map.
+ * </p>
+ * <p>
+ * In fact, the storage of actual data is a bit more complex, due to
+ * the need of splitting the data that may be too large (the value of "ID6"
+ * may be a series of 100 * 2<sup>20</sup> bytes for a 200 MiB file).
+ * Actually, files content (like ID2, ID5, ID6) and directory content
+ * values ("/", ID1, ID4) will be split and the map will look like that (
+ * <strong>note that only the files have been split for readability</strong>):
+ * </p>
+ * <table>
+ * <caption>Use of map, concrete point of view</caption>
+ * <tr>
+ * <th>Key (String)</th>
+ * <th>Value (byte[])</th>
+ * </tr>
+ * <tr>
+ * <td>"/"</td>
+ * <td>"D:my_folder:ID1\n<br />
+ * F:my_file.ext:ID2\n<br />
+ * L:my_link:ID3"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID1"</td>
+ * <td>"D:my_sub_folder:ID4\n<br />
+ * F:my_other_file.ext:ID5"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID2"</td>
+ * <td>"6;ID8"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID3"</td>
+ * <td>"/my_folder/my_sub_folder/my_deep_file.ext"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID4"</td>
+ * <td>"F:my_deep_file:ID6\n<br />
+ * D:my_empty_folder:ID7"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID5"</td>
+ * <td>"14689;ID9"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID6"</td>
+ * <td>"79874567;ID10;ID11;ID12"</td>
+ * </tr>
+ * <tr>
+ * <td>"ID7"</td>
+ * <td>""</td>
+ * </tr>
+ * <tr>
+ * <td>"ID8"</td>
+ * <td>010101</td>
+ * </tr>
+ * <tr>
+ * <td>"ID9"</td>
+ * <td>011010100111010...</td>
+ * </tr>
+ * <tr>
+ * <td>"ID10"</td>
+ * <td>110001010101011...</td>
+ * </tr>
+ * <tr>
+ * <td>"ID11"</td>
+ * <td>111010110101101...</td>
+ * </tr>
+ * <tr>
+ * <td>"ID12"</td>
+ * <td>011101100111010...</td>
+ * </tr>
+ * </table>
+ *
+ * <p>
+ * Values of ID2, ID5, ID6 are no longer the file content. It's now a set
+ * of meta-information information data called <code>metablock</code>.
+ * A metablock is composed of the total size of the file followed by
+ * an ordered lists of IDs of the different blocks composing the file.
+ * Those informations are separated by ";" (see
+ * {@link EntryUtil#BLOCKIDS_SEPARATOR}).
+ * </p>
+ * <p>
+ * In the above example:
+ * <ul>
+ * <li>
+ * "/my_file.ext" is a file which size is 6 bytes, it is
+ * composed of one block available at ID8;
+ * </li>
+ * <li>
+ * "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes)
+ * which content is available at ID9;
+ * </li>
+ * <li>
+ * "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its
+ * size is 79874567. It has been split in three blocks: ID10, ID11 and
+ * ID12.
+ * </li>
+ * </ul>
+ * </p>
+ * <p>
+ * When reading and writing in storage, split is done transparently. When
+ * reading, a Stream is returned: it loads data blocks after blocks
+ * when needed (see {@link Storage.SplitBlocksInputStream}). When writing,
+ * data are split in blocks of a maximum configurable size
+ * (see {@link org.nuiton.disworkfs.DisworkConfig}).
+ * </p>
+ */
+
+package org.nuiton.disworkfs.storage;
\ No newline at end of file
Modified: trunk/diswork-fs/src/main/resources/log4j.properties
===================================================================
--- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-25 15:24:32 UTC (rev 51)
@@ -5,4 +5,4 @@
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n
# package level
-log4j.logger.org.nuiton.disworkfs=TRACE
\ No newline at end of file
+log4j.logger.org.nuiton.disworkfs=WARN
\ No newline at end of file
Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -1,253 +0,0 @@
-package org.nuiton.disworkfs;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.nuiton.util.FileUtil;
-
-
-public class DistributedFileSystemTest {
-
- /**
- * a place to store files for the test it's a subdirectory of the OS temp
- * dir e.g. under linux /tmp/disworkfs/tests/
- */
- static protected String tempDirectoryPath = System.getProperty(
- "java.io.tmpdir", ".")
- + "/disworkfs/tests";
-
- /**
- * We will create a file at this path for test purpose
- */
- static protected String randomFilePath = tempDirectoryPath + "/randomfile";
-
- /**
- * The file will have this fixed size
- */
- static protected int randomFileSize = 10000;
-
- static protected DistributedFileSystem fileSystem;
-
- @Before
- public void setUp() throws Exception {
- File tempDirectory = new File(tempDirectoryPath);
- tempDirectory.mkdir();
-
- Random random = new Random();
- // creating random data for the file
- byte[] randomBytes = new byte[randomFileSize];
- random.nextBytes(randomBytes);
-
- // dumping random data into the file
- File randomFile = new File(randomFilePath);
- FileUtils.writeByteArrayToFile(randomFile, randomBytes);
-
- DisworkConfig disworkConfig = new DisworkConfig();
-
- fileSystem = new DisworkFileSystem(disworkConfig);
- }
-
- @After
- public void tearDown() throws Exception {
- // cleaning
- FileUtil.deleteRecursively(tempDirectoryPath);
- }
-
- @Test
- public void testWrite() throws Exception {
- fileSystem.write("/", "my_file", new FileInputStream(randomFilePath));
- }
-
- @Test
- public void testExists() throws Exception {
- fileSystem.write("/", "my_file", new FileInputStream(randomFilePath));
- assertTrue(fileSystem.exists("/my_file"));
- assertFalse(fileSystem.exists("/my_other_file"));
- }
-
- /**
- * try to read a file that as never been created nor written
- */
- @Test(expected = FileNotFoundException.class)
- public void testFailAtRead() throws Exception {
- fileSystem.read("/not_existing_file");
- }
-
- /**
- * testing {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream}
- * by storing a "-1" byte, can be buggy due to the use of read()
- * @throws IOException
- */
- @Test
- public void testSplit() throws IOException {
-
- byte[] bytes = new byte[1];
- bytes[0] = -0x1;
-
- InputStream source;
-
- source = new ByteArrayInputStream(bytes);
- fileSystem.write("/", "my_file", source);
-
- source.close();
-
-
- source = new ByteArrayInputStream(bytes);
- InputStream readResult = fileSystem.read("/my_file");
-
- int read = 0;
- byte[] b = new byte[1];
-
- read = readResult.read(b);
-
- assertEquals(1, read);
- assertArrayEquals(bytes, b);
-
- }
-
- /**
- * writing a file at the root directory and reading it.
- * finally, compare original source and read result
- * byte-to-byte : contents should be equals
- * @throws Exception
- */
- @Test
- public void testWriteRead() throws Exception {
-
- InputStream source = new FileInputStream(randomFilePath);
-
- fileSystem.write("/", "my_file", source);
-
- source.close();
-
- InputStream readResult;
-
- // to be used for debugging purpose
-
- source = new FileInputStream(randomFilePath);
- readResult = fileSystem.read("/my_file");
-
- System.out.println("source.available() = " + source.available());
- System.out.println("readResult.available() = " + readResult.available());
-
- byte[] sourceAsBytes = IOUtils.toByteArray(source);
- byte[] readResultAsBytes = IOUtils.toByteArray(readResult);
- /*
- System.out.println("source.available() = " + source.available());
- System.out.println("readResult.available() = " + readResult.available());
-
- System.out.println("source (" + sourceAsBytes.length + ") = " + Arrays.toString(sourceAsBytes));
- System.out.println("result (" + readResultAsBytes.length + ") = " + Arrays.toString(readResultAsBytes));
- */
- assertArrayEquals(sourceAsBytes, readResultAsBytes);
-
-
- source = new FileInputStream(randomFilePath);
- readResult = fileSystem.read("/my_file");
-
- boolean actualContentEquality =
- IOUtils.contentEquals(source, readResult);
- source.close();
- readResult.close();
-
- assertTrue(actualContentEquality);
-
- }
-
- /**
- * this use case should raise an exception because my_folder
- * doesn't exists
- */
- @Test(expected = IOException.class)
- public void testFailAtWrite() throws Exception {
- fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
- }
-
- @Test
- public void testMkdir() throws Exception {
- fileSystem.mkdir("/my_folder");
- assertTrue(fileSystem.exists("/my_folder"));
- fileSystem.mkdir("/my_folder/my_sub_folder");
- assertTrue(fileSystem.exists("/my_folder/my_sub_folder"));
- }
-
- @Test
- public void testWriteInFolder() throws Exception {
- fileSystem.mkdir("/my_folder");
- fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
- fileSystem.mkdir("/my_folder/my_sub_folder");
- fileSystem.write("/my_folder/my_sub_folder", "my_file", new FileInputStream(randomFilePath));
- assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file"));
- }
-
- @Test
- public void testln() throws Exception {
- fileSystem.mkdir("/my_folder");
- fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
- fileSystem.ln("/my_link", "/my_folder/my_file");
-
- InputStream source = new FileInputStream(randomFilePath);
- InputStream readResult = fileSystem.read("/my_link");
-
- boolean actualContentEquality =
- IOUtils.contentEquals(source, readResult);
- source.close();
- readResult.close();
-
- assertTrue(actualContentEquality);
- }
-
- @Test(expected = IOException.class)
- public void testFailAtLn() throws Exception {
- fileSystem.ln("/my_link", "/wrong_target_path");
- }
-
- @Test
- public void testRemove() throws Exception {
- fileSystem.mkdir("/my_folder");
- fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
- fileSystem.remove("/my_folder/my_file");
- assertTrue(fileSystem.exists("/my_folder"));
- assertFalse(fileSystem.exists("/my_folder/my_file"));
- fileSystem.remove("/my_folder");
- assertFalse(fileSystem.exists("/my_folder"));
- }
-
- @Test(expected = IOException.class)
- public void testFailAtRemove() throws Exception {
- fileSystem.mkdir("/my_folder");
- fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
-
- // trying to remove a non-empty directory should raise an exception
- fileSystem.remove("/my_folder");
- }
-
- @Test
- public void testLs() throws Exception {
- fileSystem.mkdir("/my_folder");
- fileSystem.mkdir("/my_folder/my_sub_dir");
- fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
- fileSystem.ln("/my_folder/my_link", "my_file");
-
- // trying to remove a non-empty directory should raise an exception
- List<String> lsResult = fileSystem.ls("/my_folder");
- assertEquals(3, lsResult.size());
- }
-
-}
Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java (from rev 50, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java)
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java (rev 0)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -0,0 +1,316 @@
+package org.nuiton.disworkfs;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.nuiton.util.FileUtil;
+
+
+public class DisworkFileSystemTest {
+
+ /**
+ * a place to store files for the test it's a subdirectory of the OS temp
+ * dir e.g. under linux /tmp/disworkfs/tests/
+ */
+ static protected String tempDirectoryPath =
+ System.getProperty("java.io.tmpdir", ".") // temp directory
+ + "/disworkfs/tests";
+
+ /**
+ * We will create a file at this path for test purpose
+ */
+ static protected String randomFilePath = tempDirectoryPath + "/randomfile";
+
+ /**
+ * The file will have this fixed size
+ */
+ static protected int randomFileSize = 9999;
+
+ static protected DisworkFileSystem fileSystem;
+
+ /**
+ * This setUp creates in a temp directory a file of size
+ * {@link randomFileSize} (fulfilling it with random bytes)
+ * This file can be found at {@link #randomFilePath}
+ * At the end of the method {@link #fileSystem} is ready to be used
+ * @throws Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ // create a temp directory for our test
+ File tempDirectory = new File(tempDirectoryPath);
+ tempDirectory.mkdir();
+
+ // creating random data for the file
+ Random random = new Random();
+ byte[] randomBytes = new byte[randomFileSize];
+ random.nextBytes(randomBytes);
+
+ // dumping random data into the file
+ File randomFile = new File(randomFilePath);
+ FileUtils.writeByteArrayToFile(randomFile, randomBytes);
+
+ // finally, initiate the fileSystem
+ DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig();
+ fileSystem = new DisworkFileSystem(disworkConfig);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // cleaning
+ FileUtil.deleteRecursively(tempDirectoryPath);
+ }
+
+ /**
+ * writing a file at root directory should not raise any exception
+ * @throws Exception
+ */
+ @Test
+ public void testWrite() throws Exception {
+ fileSystem.write("/", "my_file", new FileInputStream(randomFilePath));
+ }
+
+ /**
+ * First, write a file then test if exists return true.
+ * @throws Exception
+ */
+ @Test
+ public void testExists() throws Exception {
+ fileSystem.write("/", "my_file", new FileInputStream(randomFilePath));
+ assertTrue(fileSystem.exists("/my_file"));
+ assertFalse(fileSystem.exists("/my_other_file"));
+ }
+
+ /**
+ * try to read a file that as never been created nor written
+ */
+ @Test(expected = FileNotFoundException.class)
+ public void testFailAtRead() throws Exception {
+ fileSystem.read("/not_existing_file");
+ }
+
+ /**
+ * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream}
+ * by storing a "-1" byte, can be buggy due to the use of read()
+ * @throws IOException
+ */
+ @Test
+ public void testSplit() throws IOException {
+
+ byte[] bytes = new byte[1];
+ bytes[0] = -0x1;
+
+ InputStream source;
+
+ source = new ByteArrayInputStream(bytes);
+ fileSystem.write("/", "my_file", source);
+
+ source.close();
+
+
+ source = new ByteArrayInputStream(bytes);
+ InputStream readResult = fileSystem.read("/my_file");
+
+ int read = 0;
+ byte[] b = new byte[1];
+
+ read = readResult.read(b);
+
+ assertEquals(1, read);
+ assertArrayEquals(bytes, b);
+
+ }
+
+ /**
+ * writing a file at the root directory and reading it.
+ * finally, compare original source and read result
+ * byte-to-byte: contents should be equals
+ * @throws Exception
+ */
+ @Test
+ public void testWriteRead() throws Exception {
+
+ InputStream source = new FileInputStream(randomFilePath);
+
+ fileSystem.write("/", "my_file", source);
+
+ source.close();
+
+ InputStream readResult;
+
+ // now, the checks. We read the original file and the result of
+ // a read() and then compare it byte-to-byte
+
+ source = new FileInputStream(randomFilePath);
+ readResult = fileSystem.read("/my_file");
+
+ assertEquals(randomFileSize, source.available());
+ assertEquals(randomFileSize, readResult.available());
+
+ byte[] sourceAsBytes = IOUtils.toByteArray(source);
+ byte[] readResultAsBytes = IOUtils.toByteArray(readResult);
+
+ assertArrayEquals(sourceAsBytes, readResultAsBytes);
+
+ source.close();
+ readResult.close();
+
+ }
+
+ /**
+ * this use case should raise an exception because my_folder
+ * doesn't exists
+ */
+ @Test(expected = IOException.class)
+ public void testFailAtWrite() throws Exception {
+ fileSystem.write("/my_folder", "my_file",
+ new FileInputStream(randomFilePath));
+ }
+
+ /**
+ * this use case should raise an exception because writing to a file
+ * that already exists
+ */
+ @Test(expected = IOException.class)
+ public void testFailAtDoubleWrite() throws Exception {
+ InputStream source = new FileInputStream(randomFilePath);
+ fileSystem.write("/my_file", source);
+ source.close();
+
+ source = new FileInputStream(randomFilePath);
+ try {
+ fileSystem.write("/my_file", source);
+ } finally {
+ source.close();
+ }
+ }
+
+ /**
+ * This test uses mkdir to create dirs and sub-dirs
+ * @throws Exception
+ */
+ @Test
+ public void testCreateDirectory() throws Exception {
+ fileSystem.createDirectory("/my_folder");
+ assertTrue(fileSystem.exists("/my_folder"));
+ fileSystem.createDirectory("/my_folder/my_sub_folder");
+ assertTrue(fileSystem.exists("/my_folder/my_sub_folder"));
+ }
+
+ /**
+ * Create some folders with mkdir and write files in those directories
+ * @throws Exception
+ */
+ @Test
+ public void testWriteInFolder() throws Exception {
+ fileSystem.createDirectory("/my_folder");
+ fileSystem.write("/my_folder", "my_file",
+ new FileInputStream(randomFilePath));
+ fileSystem.createDirectory("/my_folder/my_sub_folder");
+ fileSystem.write("/my_folder/my_sub_folder", "my_file",
+ new FileInputStream(randomFilePath));
+ assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file"));
+ }
+
+ /**
+ * create a symbolic link to a file. This test show that we can read the
+ * file using the link
+ * @throws Exception
+ */
+ @Test
+ public void testLinking() throws Exception {
+ fileSystem.createDirectory("/my_folder");
+ fileSystem.write("/my_folder", "my_file",
+ new FileInputStream(randomFilePath));
+ fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file");
+
+ InputStream source = new FileInputStream(randomFilePath);
+ InputStream readResult = fileSystem.read("/my_link");
+
+ boolean actualContentEquality =
+ IOUtils.contentEquals(source, readResult);
+ source.close();
+ readResult.close();
+
+ assertTrue(actualContentEquality);
+ }
+
+ /**
+ * Trying to create a link to a wrong target, this sould raise an exception
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testFailAtLinking() throws Exception {
+ fileSystem.createSymbolicLink("/my_link", "/wrong_target_path");
+ }
+
+ /**
+ * Trying to remove files and directories
+ * @throws Exception
+ */
+ @Test
+ public void testRemove() throws Exception {
+ fileSystem.createDirectory("/my_folder");
+ fileSystem.write("/my_folder", "my_file",
+ new FileInputStream(randomFilePath));
+ fileSystem.delete("/my_folder/my_file");
+ assertTrue(fileSystem.exists("/my_folder"));
+ assertFalse(fileSystem.exists("/my_folder/my_file"));
+ fileSystem.delete("/my_folder");
+ assertFalse(fileSystem.exists("/my_folder"));
+ }
+
+ /**
+ * By trying to remove a non-empty directory, this test should raise an
+ * exception
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testFailAtRemove() throws Exception {
+ fileSystem.createDirectory("/my_folder");
+ fileSystem.write("/my_folder", "my_file",
+ new FileInputStream(randomFilePath));
+
+ // trying to remove a non-empty directory should raise an exception
+ fileSystem.delete("/my_folder");
+ }
+
+ /**
+ * This tests uses ls
+ */
+ @Test
+ public void testListDirectory() throws Exception {
+ fileSystem.createDirectory("/my_folder");
+ fileSystem.createDirectory("/my_folder/my_sub_dir");
+ fileSystem.write("/my_folder", "my_file",
+ new FileInputStream(randomFilePath));
+ fileSystem.createSymbolicLink("/my_folder/my_link", "my_file");
+
+ List<String> lsResult = fileSystem.readDirectory("/my_folder");
+
+ // checking that result contains all the required data
+ assertTrue(lsResult.contains("my_sub_dir"));
+ assertTrue(lsResult.contains("my_file"));
+ assertTrue(lsResult.contains("my_link"));
+
+ // ... and only those
+ assertEquals(3, lsResult.size());
+ }
+
+}
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 16:46:41 UTC (rev 50)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-25 15:24:32 UTC (rev 51)
@@ -18,6 +18,7 @@
package org.nuiton.disworkfs;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -87,24 +88,24 @@
@Test
public void testGetId() {
- assertEquals("myid1", EntryUtil.getId(directoryEntry));
- assertEquals("myid2", EntryUtil.getId(fileEntry));
- assertEquals("myid3", EntryUtil.getId(linkEntry));
+ assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry));
+ assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry));
+ assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry));
}
@Test
public void testGetName() {
- assertEquals("mydir", EntryUtil.getName(directoryEntry));
- assertEquals("myfile", EntryUtil.getName(fileEntry));
- assertEquals("mylink", EntryUtil.getName(linkEntry));
+ assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry));
+ assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry));
+ assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry));
}
@Test
public void testGetType() {
- assertEquals(EntryUtil.TYPE.D, EntryUtil.getType(directoryEntry));
- assertEquals(EntryUtil.TYPE.F, EntryUtil.getType(fileEntry));
- assertEquals(EntryUtil.TYPE.L, EntryUtil.getType(linkEntry));
+ assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry));
+ assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry));
+ assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry));
}
@Test
@@ -123,7 +124,7 @@
@Test
public void testBytesToArray() {
- String s = "abcdefg@^:éèvwxyz,!;*$";
+ String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$";
String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s));
assertEquals(s, copy);
}
@@ -137,10 +138,7 @@
public void testGetBlockIdsFromMetaBlock() {
String[] expectedBlocksIds = {"myid1", "myid2", "myid3"};
String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock);
- assertEquals(expectedBlocksIds.length, actualBlocksIds.length);
- assertEquals(expectedBlocksIds[0], actualBlocksIds[0]);
- assertEquals(expectedBlocksIds[1], actualBlocksIds[1]);
- assertEquals(expectedBlocksIds[2], actualBlocksIds[2]);
+ assertArrayEquals(expectedBlocksIds, actualBlocksIds);
}
@Test
1
0
r50 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 21 May '10
by bleny＠users.nuiton.org 21 May '10
21 May '10
Author: bleny
Date: 2010-05-21 18:46:41 +0200 (Fri, 21 May 2010)
New Revision: 50
Url: http://nuiton.org/repositories/revision/diswork/50
Log:
ls + test
Added:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java
Removed:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -3,13 +3,17 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
public interface DistributedFileSystem {
public boolean exists(String path) throws IOException;
- public void write(String parent, String file, InputStream source) throws IOException;
+ public void write(String path, InputStream source) throws IOException;
+ public void write(String parent, String fileName, InputStream source)
+ throws IOException;
+
public InputStream read(String path) throws FileNotFoundException,
IOException;
@@ -19,4 +23,6 @@
public void remove(String path) throws IOException;
+ public List<String> ls(String path) throws IOException;
+
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -1,21 +1,14 @@
package org.nuiton.disworkfs;
-import java.io.File;
-import java.util.Random;
-
import org.nuiton.util.ApplicationConfig;
public class DisworkConfig extends ApplicationConfig {
public DisworkConfig() {
- Random random = new Random();
- setDefaultOption("storage", "/tmp/disworkfs/storage" + random.nextInt());
-
- // replication strategy...
- // TODO 20100519 bleny chunk size parameter
+ setDefaultOption("blocks_size", "10485760"); // 10 MiB
}
- public File getStoragePath() {
- return getOptionAsFile("storage");
+ public int getBlockSize() {
+ return Integer.parseInt(getOption("blocks_size"));
}
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -3,10 +3,13 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.nuiton.disworkfs.storage.EntryUtil;
+import org.nuiton.disworkfs.storage.InMemoryMap;
import org.nuiton.disworkfs.storage.Storage;
public class DisworkFileSystem implements DistributedFileSystem {
@@ -16,7 +19,7 @@
private static final Log log = LogFactory.getLog(DisworkFileSystem.class);
public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException {
- storage = new Storage(new InMemoryMap());
+ storage = new Storage(disworkConfig, new InMemoryMap());
storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent());
}
@@ -40,7 +43,8 @@
}
// FIXME 20105021 bleny works fine but is not understandable
- protected String walk(String path, String current, String content) throws IOException {
+ protected String walk(String path, String current, String content)
+ throws IOException {
String result = null;
if (path.equals("/")) {
@@ -49,6 +53,7 @@
+ "/";
}
+
String parentPath = EntryUtil.getParentFromPath(path);
if (content == null) {
@@ -150,6 +155,13 @@
}
@Override
+ public void write(String path, InputStream source) throws IOException {
+ String parent = EntryUtil.getParentFromPath(path);
+ String name = EntryUtil.getNameFromPath(path);
+ write(parent, name, source);
+ }
+
+ @Override
public void write(String parent, String fileName, InputStream source) throws IOException {
String entryParent = walk(parent);
@@ -215,6 +227,14 @@
}
public void ln(String parent, String name, String target) throws IOException {
+
+ if (target.startsWith("/")) {
+ // target is absolute
+ } else {
+ // target is relative, taking this in consideration
+ target = EntryUtil.resolveLink(parent, target);
+ }
+
if (exists(target)) {
String entryParent = walk(parent);
@@ -290,4 +310,33 @@
}
}
+ @Override
+ public List<String> ls(String path) throws IOException {
+ String entry = walk(path);
+ List<String> result = null;
+
+
+ if (entry == null) {
+ throw new IOException(path + " doesn't exists");
+ } else {
+ if (EntryUtil.isLink(entry)) {
+ String target = storage.getLink(EntryUtil.getId(entry));
+ entry = walk(target);
+ }
+
+ if (EntryUtil.isDirectory(entry)) {
+ String content = storage.getDirectory(EntryUtil.getId(entry));
+ String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR);
+ result = new ArrayList<String>();
+ for (String elementEntry : entries) {
+ result.add(EntryUtil.getName(elementEntry));
+ }
+ } else {
+ throw new IOException(path + " is not a directory");
+ }
+ }
+
+ return result;
+ }
+
}
Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -1,20 +0,0 @@
-package org.nuiton.disworkfs;
-
-import java.util.Arrays;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-public class InMemoryMap extends HashMap<String, byte[]> {
-
- private final Log log = LogFactory.getLog(InMemoryMap.class);
-
- @Override
- public byte[] put(String key, byte[] value) {
- log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")");
- byte[] valueCopy = value.clone();
- log.info("putting " + valueCopy.length + " bytes");
- return super.put(key, valueCopy);
- }
-}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -1,6 +1,7 @@
package org.nuiton.disworkfs.storage;
+import java.nio.charset.Charset;
import java.util.UUID;
import org.apache.commons.io.FilenameUtils;
@@ -27,6 +28,8 @@
static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/};
+ static protected Charset CHARSET = Charset.forName("UTF-8");
+
/**
* generate a new Id usage as a new key for the map
* @return a String containing the id
@@ -87,14 +90,14 @@
}
static public int getTotalSizeFromMetaBlock(String metaBlock) {
- // FIXME 20100520 bleny this workaround should not be
+ // dealing with empty meta-block (for empty directory or file)
if (metaBlock.equals("0")) return 0;
String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR));
return Integer.valueOf(result);
}
static public String[] getBlockIdsFromMetaBlock(String metaBlock) {
- // FIXME 20100520 bleny this workaround should not be
+ // dealing with empty meta-block (for empty directory or file)
if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) {
return new String[0];
}
@@ -188,7 +191,7 @@
public static byte[] stringToBytes(String string) {
// FIXME 20100519 bleny this code may cause bad conversion
// due to charsets
- byte[] bytes = string.getBytes();
+ byte[] bytes = string.getBytes(CHARSET);
return bytes;
}
@@ -198,7 +201,7 @@
* @return a byte array containing all the bytes in <code>string</code>
*/
public static String bytesToString(byte[] bytes) {
- return new String(bytes);
+ return new String(bytes, CHARSET);
}
public static String getParentFromPath(String path) {
Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java (from rev 47, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java)
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java (rev 0)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -0,0 +1,20 @@
+package org.nuiton.disworkfs.storage;
+
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class InMemoryMap extends HashMap<String, byte[]> {
+
+ private final Log log = LogFactory.getLog(InMemoryMap.class);
+
+ @Override
+ public byte[] put(String key, byte[] value) {
+ log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")");
+ byte[] valueCopy = value.clone();
+ log.info("putting " + valueCopy.length + " bytes");
+ return super.put(key, valueCopy);
+ }
+}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -8,6 +8,7 @@
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.nuiton.disworkfs.DisworkConfig;
public class Storage {
@@ -84,9 +85,12 @@
}
protected Map<String, byte[]> map;
+
+ protected DisworkConfig disworkConfig;
- public Storage(Map<String, byte[]> map) {
+ public Storage(DisworkConfig disworkConfig, Map<String, byte[]> map) {
this.map = map;
+ this.disworkConfig = disworkConfig;
}
public boolean contains(Object id) {
@@ -167,8 +171,8 @@
int read = 0;
int totalSize = 0;
- // TODO 20100519 bleny the size of blocks should be a config directive
- byte[] buffer = new byte[10 * 1024 * 1024];
+ int bufferSize = disworkConfig.getBlockSize();
+ byte[] buffer = new byte[bufferSize];
while ((read = value.read(buffer)) != -1) {
totalSize += read;
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -11,21 +11,18 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.nuiton.util.FileUtil;
-public class DistributedFileSystemTest {
-
- private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class);
+public class DistributedFileSystemTest {
/**
* a place to store files for the test it's a subdirectory of the OS temp
@@ -240,5 +237,17 @@
// trying to remove a non-empty directory should raise an exception
fileSystem.remove("/my_folder");
}
+
+ @Test
+ public void testLs() throws Exception {
+ fileSystem.mkdir("/my_folder");
+ fileSystem.mkdir("/my_folder/my_sub_dir");
+ fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
+ fileSystem.ln("/my_folder/my_link", "my_file");
+
+ // trying to remove a non-empty directory should raise an exception
+ List<String> lsResult = fileSystem.ls("/my_folder");
+ assertEquals(3, lsResult.size());
+ }
}
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:14:22 UTC (rev 49)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 16:46:41 UTC (rev 50)
@@ -4,33 +4,23 @@
import java.util.Arrays;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
+import org.nuiton.disworkfs.storage.InMemoryMap;
public class InMemoryMapTest {
- private final Log log = LogFactory.getLog(InMemoryMapTest.class);
-
-
- @Before
- public void setUp() throws Exception {}
- @After
- public void tearDown() throws Exception {}
-
+ /**
+ * this test show that the InMemoryMap put implies a copy of the data.
+ * After the put, the original value is modified. A final check shows
+ * there is no side effect
+ */
@Test
public void testPut() {
InMemoryMap map = new InMemoryMap();
- byte[] expected = {0x1, 0x2};
-
+ byte[] expected = {0x1, 0x2};
map.put("key", expected);
-
expected[0] = 0xf;
-
byte[] actual = map.get("key");
-
assertFalse(Arrays.equals(expected, actual));
}
1
0
r49 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 21 May '10
by bleny＠users.nuiton.org 21 May '10
21 May '10
Author: bleny
Date: 2010-05-21 17:14:22 +0200 (Fri, 21 May 2010)
New Revision: 49
Url: http://nuiton.org/repositories/revision/diswork/49
Log:
clean imports
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:04:11 UTC (rev 48)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:14:22 UTC (rev 49)
@@ -3,7 +3,6 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:04:11 UTC (rev 48)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:14:22 UTC (rev 49)
@@ -1,9 +1,9 @@
package org.nuiton.disworkfs;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertArrayEquals;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -11,7 +11,6 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Arrays;
import java.util.Random;
import org.apache.commons.io.FileUtils;
@@ -23,10 +22,7 @@
import org.junit.Test;
import org.nuiton.util.FileUtil;
-import com.sun.org.apache.xml.internal.security.keys.storage.StorageResolverException;
-import com.sun.xml.internal.ws.util.ByteArrayBuffer;
-
public class DistributedFileSystemTest {
private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class);
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:04:11 UTC (rev 48)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:14:22 UTC (rev 49)
@@ -1,5 +1,7 @@
package org.nuiton.disworkfs;
+import static org.junit.Assert.assertFalse;
+
import java.util.Arrays;
import org.apache.commons.logging.Log;
@@ -8,8 +10,6 @@
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertFalse;
-
public class InMemoryMapTest {
private final Log log = LogFactory.getLog(InMemoryMapTest.class);
1
0
r48 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org 21 May '10
by bleny＠users.nuiton.org 21 May '10
21 May '10
Author: bleny
Date: 2010-05-21 17:04:11 +0200 (Fri, 21 May 2010)
New Revision: 48
Url: http://nuiton.org/repositories/revision/diswork/48
Log:
ln, remove with tests
Modified:
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 13:01:45 UTC (rev 47)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 15:04:11 UTC (rev 48)
@@ -15,4 +15,8 @@
public void mkdir(String path) throws IOException;
+ public void ln(String path, String target) throws IOException;
+
+ public void remove(String path) throws IOException;
+
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 13:01:45 UTC (rev 47)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:04:11 UTC (rev 48)
@@ -3,6 +3,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -206,5 +207,88 @@
throw new IOException(parent + " is not a directory");
}
}
+
+ @Override
+ public void ln(String path, String target) throws IOException {
+ String parent = EntryUtil.getParentFromPath(path);
+ String name = EntryUtil.getNameFromPath(path);
+ ln(parent, name, target);
+ }
+ public void ln(String parent, String name, String target) throws IOException {
+ if (exists(target)) {
+
+ String entryParent = walk(parent);
+
+ if (entryParent == null) {
+ throw new IOException(parent + " directory doesn't exists");
+ }
+
+ if (EntryUtil.isDirectory(entryParent)) {
+ String parentId = EntryUtil.getId(entryParent);
+ String content = storage.getDirectory(parentId);
+ String newLinkId = EntryUtil.generateId();
+ String newContent = EntryUtil.addEntryToDirectoryContent(
+ content, EntryUtil.TYPE.L, name, newLinkId);
+
+ // store link before meta info
+ storage.putLink(newLinkId, target);
+ // update meta info directory
+ storage.putDirectory(parentId, newContent);
+ } else {
+ throw new IOException(parent + " is not a directory");
+ }
+
+
+ } else {
+ throw new IOException(target + " is not a valid target");
+ }
+ }
+
+ @Override
+ public void remove(String path) throws IOException {
+ String parent = EntryUtil.getParentFromPath(path);
+ String name = EntryUtil.getNameFromPath(path);
+ log.info("trying to remove " + path);
+ remove(parent, name);
+ }
+
+ protected void remove(String parent, String name) throws IOException {
+ String entryParent = walk(parent);
+
+ if (entryParent == null) {
+ throw new IOException(parent + " directory doesn't exists");
+ }
+
+ if (EntryUtil.isDirectory(entryParent)) {
+ String parentId = EntryUtil.getId(entryParent);
+ String content = storage.getDirectory(parentId);
+
+ String entry = EntryUtil.findEntryInDirectory(content, name);
+ String idToRemove = EntryUtil.getId(entry);
+
+ String newContent = EntryUtil.removeEntryFromEntries(
+ content, name);
+
+ // check if not removing a non-empty directory
+ if (EntryUtil.isDirectory(entry)) {
+ String innerDirectoryId = EntryUtil.getId(entry);
+ String innerDirectoryContent = storage.getDirectory(innerDirectoryId);
+ if (! innerDirectoryContent.equals(EntryUtil.newEmptyDirectoryContent())) {
+ // directory is not empty
+ throw new IOException("trying to remove a non-empty directory");
+ }
+ }
+
+
+ // update meta info directory
+ storage.putDirectory(parentId, newContent);
+ // store file before meta info
+ storage.remove(idToRemove);
+
+ } else {
+ throw new IOException(parent + " is not a directory");
+ }
+ }
+
}
Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java
===================================================================
--- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 13:01:45 UTC (rev 47)
+++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 15:04:11 UTC (rev 48)
@@ -212,5 +212,11 @@
public static String getNameFromPath(String path) {
return FilenameUtils.getName(path);
}
+
+ public static String removeEntryFromEntries(String content, String name) {
+ String entry = findEntryInDirectory(content, name);
+ String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, "");
+ return newContent;
+ }
}
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 13:01:45 UTC (rev 47)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:04:11 UTC (rev 48)
@@ -182,7 +182,7 @@
* doesn't exists
*/
@Test(expected = IOException.class)
- public void testWriteFail() throws Exception {
+ public void testFailAtWrite() throws Exception {
fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
}
@@ -203,5 +203,46 @@
assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file"));
}
+ @Test
+ public void testln() throws Exception {
+ fileSystem.mkdir("/my_folder");
+ fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
+ fileSystem.ln("/my_link", "/my_folder/my_file");
+
+ InputStream source = new FileInputStream(randomFilePath);
+ InputStream readResult = fileSystem.read("/my_link");
+
+ boolean actualContentEquality =
+ IOUtils.contentEquals(source, readResult);
+ source.close();
+ readResult.close();
+
+ assertTrue(actualContentEquality);
+ }
+
+ @Test(expected = IOException.class)
+ public void testFailAtLn() throws Exception {
+ fileSystem.ln("/my_link", "/wrong_target_path");
+ }
+
+ @Test
+ public void testRemove() throws Exception {
+ fileSystem.mkdir("/my_folder");
+ fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
+ fileSystem.remove("/my_folder/my_file");
+ assertTrue(fileSystem.exists("/my_folder"));
+ assertFalse(fileSystem.exists("/my_folder/my_file"));
+ fileSystem.remove("/my_folder");
+ assertFalse(fileSystem.exists("/my_folder"));
+ }
+
+ @Test(expected = IOException.class)
+ public void testFailAtRemove() throws Exception {
+ fileSystem.mkdir("/my_folder");
+ fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath));
+
+ // trying to remove a non-empty directory should raise an exception
+ fileSystem.remove("/my_folder");
+ }
}
Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java
===================================================================
--- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 13:01:45 UTC (rev 47)
+++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 15:04:11 UTC (rev 48)
@@ -142,4 +142,20 @@
assertEquals(expectedBlocksIds[1], actualBlocksIds[1]);
assertEquals(expectedBlocksIds[2], actualBlocksIds[2]);
}
+
+ @Test
+ public void testRemoveEntryFromEntries() {
+
+ String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
+ + fileEntry + EntryUtil.ENTRIES_SEPARATOR
+ + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
+
+ String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR
+ + linkEntry + EntryUtil.ENTRIES_SEPARATOR;
+
+ String actual = EntryUtil.removeEntryFromEntries(content, "myfile");
+
+ assertEquals(expected, actual);
+
+ }
}
1
0