r56 - in trunk: . diswork-fs src/site src/site/rst/diswork-fs
Author: bleny Date: 2010-06-02 15:08:02 +0200 (Wed, 02 Jun 2010) New Revision: 56 Url: http://nuiton.org/repositories/revision/diswork/56 Log: cleaning pom, some documentation Added: trunk/src/site/rst/diswork-fs/how_it_works.rst Modified: trunk/diswork-fs/pom.xml trunk/pom.xml trunk/src/site/site_en.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-02 12:56:07 UTC (rev 55) +++ trunk/diswork-fs/pom.xml 2010-06-02 13:08:02 UTC (rev 56) @@ -1,62 +1,48 @@ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.nuiton</groupId> - <artifactId>diswork</artifactId> - <version>0.0.1-SNAPSHOT</version> - </parent> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.nuiton</groupId> + <artifactId>diswork</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> - <groupId>org.nuiton.diswork</groupId> - <artifactId>diswork-fs</artifactId> + <groupId>org.nuiton.diswork</groupId> + <artifactId>diswork-fs</artifactId> - <packaging>jar</packaging> - <name>disworkfs</name> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>2.0.2</version> - <configuration> - <source>1.6</source> - <target>1.6</target> - </configuration> - </plugin> - </plugins> - </build> - <dependencies> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>org.nuiton</groupId> - <artifactId>nuiton-utils</artifactId> - </dependency> - <dependency> - <groupId>org.freepastry</groupId> - <artifactId>pastry</artifactId> - </dependency> - <dependency> - <groupId>fr.inria.peerunit</groupId> - <artifactId>PeerUnit</artifactId> - </dependency> + <packaging>jar</packaging> + <name>disworkfs</name> + <dependencies> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + </dependency> + <dependency> + <groupId>org.freepastry</groupId> + <artifactId>pastry</artifactId> + </dependency> + <dependency> + <groupId>fr.inria.peerunit</groupId> + <artifactId>PeerUnit</artifactId> + </dependency> - <!-- test --> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> -</project> - + <!-- test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-02 12:56:07 UTC (rev 55) +++ trunk/pom.xml 2010-06-02 13:08:02 UTC (rev 56) @@ -1,11 +1,12 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> + <modelVersion>4.0.0</modelVersion> - <!-- ************************************************************* --> - <!-- *** POM Relationships *************************************** --> - <!-- ************************************************************* --> + <!-- ************************************************************* --> + <!-- *** POM Relationships *************************************** --> + <!-- ************************************************************* --> <parent> <groupId>org.nuiton</groupId> @@ -13,176 +14,183 @@ <version>2.1.5</version> </parent> - <artifactId>diswork</artifactId> - <version>0.0.1-SNAPSHOT</version> + <artifactId>diswork</artifactId> + <version>0.0.1-SNAPSHOT</version> - <modules> - <module>diswork-fs</module> - </modules> + <modules> + <module>diswork-fs</module> + </modules> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - <version>1.1.1</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <version>1.2.14</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>jgroups</groupId> - <artifactId>jgroups</artifactId> - <version>2.9.0.GA</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>1.4</version> - <type>jar</type> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>2.5</version> - <type>jar</type> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.nuiton</groupId> - <artifactId>nuiton-utils</artifactId> - <version>1.2.2</version> - <type>jar</type> - <scope>compile</scope> - </dependency> -<!-- - <dependency> - <groupId>fr.inria.peerunit</groupId> - <artifactId>PeerUnit</artifactId> - <version>1.0</version> - <scope>compile</scope> - </dependency> ---> - <!-- test --> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.8.1</version> - <scope>test</scope> - </dependency> - </dependencies> - </dependencyManagement> - <!-- ************************************************************* --> - <!-- *** Project Information ************************************* --> - <!-- ************************************************************* --> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.14</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>jgroups</groupId> + <artifactId>jgroups</artifactId> + <version>2.9.0.GA</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>1.4</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.5</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + <version>1.2.2</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>fr.inria.peerunit</groupId> + <artifactId>PeerUnit</artifactId> + <version>1.0</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.kth</groupId> + <artifactId>dks</artifactId> + <version>0.0.1</version> + </dependency> + <dependency> + <groupId>org.freepastry</groupId> + <artifactId>pastry</artifactId> + <version>2.1</version> + </dependency> + + <!-- test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.8.1</version> + <scope>test</scope> + </dependency> + </dependencies> + </dependencyManagement> + <!-- ************************************************************* --> + <!-- *** Project Information ************************************* --> + <!-- ************************************************************* --> - <name>Diswork</name> - <description>Distributed computing application</description> - <inceptionYear>2010</inceptionYear> - <url>http://maven-site.nuiton.org/diswork</url> + <name>Diswork</name> + <description>Distributed computing application</description> + <inceptionYear>2010</inceptionYear> + <url>http://maven-site.nuiton.org/diswork</url> - <developers> - <developer> - <name>Brendan Le Ny</name> - <id>bleny</id> - <email>leny@codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - <developer> - <name>Benjamin Poussin</name> - <id>bpoussin</id> - <email>poussin@codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - <role>Debian packager</role> - </roles> - </developer> - <developer> - <name>Eric Chatellier</name> - <id>echatellier</id> - <email>chatellier@codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - <developer> - <name>Jean Couteau</name> - <id>jcouteau</id> - <email>couteau@codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - <developer> - <name>Tony Chemit</name> - <id>tchemit</id> - <email>chemit@codelutin.com</email> - <organization>CodeLutin</organization> - <timezone>+2</timezone> - <roles> - <role>Développeur</role> - </roles> - </developer> - </developers> + <developers> + <developer> + <name>Brendan Le Ny</name> + <id>bleny</id> + <email>leny@codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + <developer> + <name>Benjamin Poussin</name> + <id>bpoussin</id> + <email>poussin@codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + <role>Debian packager</role> + </roles> + </developer> + <developer> + <name>Eric Chatellier</name> + <id>echatellier</id> + <email>chatellier@codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + <developer> + <name>Jean Couteau</name> + <id>jcouteau</id> + <email>couteau@codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + <developer> + <name>Tony Chemit</name> + <id>tchemit</id> + <email>chemit@codelutin.com</email> + <organization>CodeLutin</organization> + <timezone>+2</timezone> + <roles> + <role>Développeur</role> + </roles> + </developer> + </developers> - <!-- ************************************************************* --> - <!-- *** Build Settings ****************************************** --> - <!-- ************************************************************* --> + <!-- ************************************************************* --> + <!-- *** Build Settings ****************************************** --> + <!-- ************************************************************* --> - <packaging>pom</packaging> + <packaging>pom</packaging> - <properties> - <!-- name of redmine project --> - <projectId>diswork</projectId> - - <!-- locales to use in documentation --> - <locales>en</locales> - </properties> + <properties> + <!-- name of redmine project --> + <projectId>diswork</projectId> - <build> - + <!-- locales to use in documentation --> + <locales>en</locales> + </properties> - <pluginManagement> - <plugins> - <plugin> - <artifactId>maven-site-plugin</artifactId> - <dependencies> - <dependency> - <groupId>org.nuiton.jrst</groupId> - <artifactId>doxia-module-jrst</artifactId> - <version>${jrst.version}</version> - </dependency> - </dependencies> - </plugin> - </plugins> - </pluginManagement> - </build> + <build> + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-site-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.nuiton.jrst</groupId> + <artifactId>doxia-module-jrst</artifactId> + <version>${jrst.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </pluginManagement> + </build> - <!-- ************************************************************* --> - <!-- *** Build Environment ************************************** --> - <!-- ************************************************************* --> + <!-- ************************************************************* --> + <!-- *** Build Environment ************************************** --> + <!-- ************************************************************* --> - <!-- Source control management. --> - <scm> - <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection> - <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection> - <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url> - </scm> + <!-- Source control management. --> + <scm> + <connection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</connection> + <developerConnection>scm:svn:http://svn.nuiton.org/svn/diswork/trunk</developerConnection> + <url>http://www.nuiton.org/repositories/browse/diswork/trunk</url> + </scm> </project> Added: trunk/src/site/rst/diswork-fs/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-fs/how_it_works.rst (rev 0) +++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-02 13:08:02 UTC (rev 56) @@ -0,0 +1,125 @@ +How Diswork File System works +============================= + +Diswork File System uses a map to store data persistently. + +We illustrate the use of a file system by setting the content of "/", +the root directory as follow: + +* a directory named "my_folder" that contains: + + - a directory named "my_sub_folder" that contains: + + + a file name "my_deep_file.ext" + + + an empty folder named "my_empty_folder" + + - a file named "my_other_file.ext" + +* a file named "my_file.ext" (containing binary data: 010101) + +* a symbolic link named "my_link" which target the path + "/my_folder/my_sub_folder/my_deep_file.ext" + +Storage uses a Map to store data. + ++--------------+---------------------------------------------+ +| Key (String) | Value (byte[]) | ++==============+=============================================+ +| | "D:my_folder:ID1\n | +| | F:my_file.ext:ID2\n | +| "/" | L:my_link:ID3" | ++--------------+---------------------------------------------+ +| | "D:my_sub_folder:ID4\n | +| "ID1" | F:my_other_file.ext:ID5" | ++--------------+---------------------------------------------+ +| "ID2" | 010101 | ++--------------+---------------------------------------------+ +| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | ++--------------+---------------------------------------------+ +| | "F:my_deep_file:ID6\n | +| "ID4" | D:my_empty_folder:ID7" | ++--------------+---------------------------------------------+ +| "ID5" | 011010100111010... | ++--------------+---------------------------------------------+ +| "ID6" | 1100010101010110110010... | ++--------------+---------------------------------------------+ +| "ID7" | "" | ++--------------+---------------------------------------------+ + +An entry is, for example, "D:my_folder:ID1". It contains the type of the +entry (<strong>D</strong>irectory, <strong>F</strong>ile or +<strong>L</strong>ink), the name of the element, and an ID to be used as +a key on the map to get the actual content. Those three informations are +separated by ":" which is EntryUtil.ENTRY_SEPARATOR. + +A directory way have multiple <em>entries</em>. Entries of a directory are +separated by "\n" which is EntryUtil.ENTRIES_SEPARATOR. + +The above description shows the main principle used to store a tree +structure in a map. + +In fact, the storage of actual data is a bit more complex, due to +the need of splitting the data that may be too large (the value of "ID6" +may be a series of 1002<sup>20</sup> bytes for a 200 MiB file). +Actually, files content (like ID2, ID5, ID6) and directory content +values ("/", ID1, ID4) will be split and the map will look like that ( +<strong>note that only the files have been split for readability</strong>): + ++--------------+---------------------------------------------+ +| Key (String) | Value (byte[]) | ++==============+=============================================+ +| | "D:my_folder:ID1\n | +| | F:my_file.ext:ID2\n | +| "/" | L:my_link:ID3" | ++--------------+---------------------------------------------+ +| | "D:my_sub_folder:ID4\n | +| "ID1" | F:my_other_file.ext:ID5" | ++--------------+---------------------------------------------+ +| "ID2" | "6;ID8" | ++--------------+---------------------------------------------+ +| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | ++--------------+---------------------------------------------+ +| | "F:my_deep_file:ID6\n | +| "ID4" | D:my_empty_folder:ID7" | ++--------------+---------------------------------------------+ +| "ID5" | "14689;ID9" | ++--------------+---------------------------------------------+ +| "ID6" | "79874567;ID10;ID11;ID12" | ++--------------+---------------------------------------------+ +| "ID7" | "" | ++--------------+---------------------------------------------+ +| "ID8" | 010101 | ++--------------+---------------------------------------------+ +| "ID9" | 011010100111010... | ++--------------+---------------------------------------------+ +| "ID10" | 110001010101011... | ++--------------+---------------------------------------------+ +| "ID11" | 111010110101101... | ++--------------+---------------------------------------------+ +| "ID12" | 011101100111010... | ++--------------+---------------------------------------------+ + +Values of ID2, ID5, ID6 are no longer the file content. It's now a set +of meta-information information data called metablock. +A metablock is composed of the total size of the file followed by +an ordered lists of IDs of the different blocks composing the file. +Those informations are separated by ";" (see +{@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). + +In the above example: + +* "/my_file.ext" is a file which size is 6 bytes, it is + composed of one block available at ID8; + +* "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes) + which content is available at ID9; + +* "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its + size is 79874567. It has been split in three blocks: ID10, ID11 and + ID12. + +When reading and writing in storage, split is done transparently. When +reading, a Stream is returned: it loads data blocks after blocks +when needed inner class Storage.SplitBlocksInputStream}). +When writing, data are split in blocks of a maximum configurable size. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-02 12:56:07 UTC (rev 55) +++ trunk/src/site/site_en.xml 2010-06-02 13:08:02 UTC (rev 56) @@ -29,6 +29,7 @@ <menu name="diswork FS"> <item href="diswork-fs/history.html" name="Install"/> + <item href="diswork-fs/how_it_works.html" name="How it works"/> </menu> <menu ref="reports"/>
participants (1)
-
bleny＠users.nuiton.org