- Diswork-commits - mailman.cloud.codelutin.com

r67 - in trunk: diswork-daemon src/site/rst/diswork-daemon
by bleny＠users.nuiton.org June 8, 2010

June 8, 2010

Author: bleny Date: 2010-06-08 15:03:59 +0200 (Tue, 08 Jun 2010) New Revision: 67 Url: http://nuiton.org/repositories/revision/diswork/67 Log: maj specif Modified: trunk/diswork-daemon/ trunk/diswork-daemon/pom.xml trunk/src/site/rst/diswork-daemon/how_it_works.rst Property changes on: trunk/diswork-daemon ___________________________________________________________________ Added: svn:ignore + target Modified: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-daemon/pom.xml 2010-06-08 13:03:59 UTC (rev 67) @@ -34,6 +34,10 @@ <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> + <dependency> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + </dependency>  <dependency> Modified: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 13:03:59 UTC (rev 67) @@ -18,10 +18,19 @@ To create a job, a node should : +#. Check that the application in the good version is available at + /bin/application/application-version.jar + #. Create a directory in home for each of his jobs /home/ID/job-1, /home/ID/job-2... -#. In a job directory, put job.sh, a command line to run. +#. In a job directory, put job.jsdl, a command line to run. This file contains + necessary information like : + + * The name of the job + * The application name, version + * File staging (needed file and where to get them) + * ... (to be completed) #. Declare the job as available by adding a symlink in /var/jobs/TODO with the link named after the current date, the id of the job owner and

1 0

r66 - in trunk: diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage src/site src/site/rst/diswork-daemon src/site/rst/diswork-fs
by bleny＠users.nuiton.org June 8, 2010

June 8, 2010

Author: bleny Date: 2010-06-08 14:13:22 +0200 (Tue, 08 Jun 2010) New Revision: 66 Url: http://nuiton.org/repositories/revision/diswork/66 Log: bugfix; documentation; specif de diswork daemon Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/src/site/rst/diswork-daemon/how_it_works.rst trunk/src/site/rst/diswork-fs/history.rst trunk/src/site/rst/diswork-fs/how_it_works.rst trunk/src/site/site_en.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-08 12:13:22 UTC (rev 66) @@ -521,9 +521,8 @@ if (EntryUtil.isLink(entry)) { String target = storage.getLink( EntryUtil.getIdFromEntry(entry)); - entry = walk(target); - } - if (EntryUtil.isDirectory(entry)) { + return readDirectory(target); + } else if (EntryUtil.isDirectory(entry)) { result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); @@ -536,8 +535,11 @@ result.add(EntryUtil.getNameFromEntry(elementEntry)); } } + } else if (EntryUtil.isFile(entry)) { + throw new IOException(path + " is not a directory but a file"); } else { - throw new IOException(path + " is not a directory"); + log.warn("strange entry" + entry); + throw new IOException("strange entry" + entry); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-08 12:13:22 UTC (rev 66) @@ -341,6 +341,20 @@ readResult = fileSystem.readDirectory("/otherdir"); assertEquals(1, readResult.size()); assertTrue(readResult.contains("finaldir")); + + // tests that delete remove the link itself and not the target + fileSystem.delete("/link/subsubdirlink"); + assertTrue(fileSystem.exists("/otherdir")); + fileSystem.delete("/link"); + assertTrue(fileSystem.exists("/dir/subdir")); + + // testing the use of multiple links at the end of a path + fileSystem.createSymbolicLink("/link1", "/dir"); + fileSystem.createSymbolicLink("/link2", "/link1"); + fileSystem.createSymbolicLink("/link3", "/link2"); + readResult = fileSystem.readDirectory("/link3"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subdir")); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) @@ -11,7 +11,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.nuiton.disworkfs.DisworkFileSystemConfig; public abstract class AbstractDisworkMapTest { @@ -21,9 +20,13 @@ protected DisworkMap map1; protected DisworkMap map2; + /** + * should be overridden by method that set map1 and map2 with different + * implementation of DisworkMap + * @throws Exception + */ @Before - public void setUp() throws Exception { - + public void setUp() throws Exception { } @After @@ -40,7 +43,7 @@ * there is no side effect */ @Test - public void testPut() { + public void testPutMakeCopy() { byte[] expected = {0x1, 0x2}; map1.put("key", expected); expected[0] = 0xf; @@ -50,11 +53,7 @@ @Test - public void test() throws Exception { - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - - map1 = new KademliaDisworkMap(config); - + public void testPutGet() throws Exception { byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; map1.put("test", newBytes); @@ -65,12 +64,7 @@ @Test public void test2nodes() throws Exception { - DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); - map1 = new KademliaDisworkMap(config1); - DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); - map2 = new KademliaDisworkMap(config2); - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; log.info("trying put test"); Modified: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,4 +1,82 @@ +======================== How Diswork Daemon works ======================== -Diswork Daemon uses Diswork File System. \ No newline at end of file +Overview +-------- + +Diswork Daemon uses Diswork File System. + +Submit a job +------------ + +First, a node, before joining a network generate, for himself, a unique ID. And +then, create a directory "/home/ID". If this is not the first time the node +is run, ID should be reused so jobs can be found back after disconnect/connect. + +In his home directory, a node can put and remove his jobs. + +To create a job, a node should : + +#. Create a directory in home for each of his jobs + /home/ID/job-1, /home/ID/job-2... + +#. In a job directory, put job.sh, a command line to run. + +#. Declare the job as available by adding a symlink in /var/jobs/TODO + with the link named after the current date, the id of the job owner and + the job name. Places possible in /var/jobs/ are : + +#. Check the content of /home/ID/job-i/status + +#. Finally, clean home by deleting the directories of finished jobs + +Perform a job +------------- + +To perform a job, a node should find a job in /var/jobs/, this directory has +6 sub-directories : + +- /var/jobs/TODO + +- /var/jobs/FAILED_1 + +- /var/jobs/FAILED_2 + +- /var/jobs/FAILED_3 + +- /var/jobs/DONE + +TODO, FAILED_1, FAILED_2 all have a sub-directory RUNNING. + +#. Choose a job in one of those directories except DONE and FAILED_3. Use the + data in the link same to know which job should be done first; + +#. Once the job is chosen, move the symlink to the RUNNING subdir where you + found the link. Then, follow the link to get the actual datas needed for + the job; + +#. Run the job; + +#. Now there is two way to proceed : + + - If the job successfully ran (no exception raised and return code is 0) : + + #. write the results in the job directory; + + #. move the link to DONE. + + - If the job has failed (a job taking too much time may be considered as + a failure). move it to according to where the job was found. If + the job was in TODO, move it to FAILED_1 (read "failed once") ; + if the job was in FAILED_1, move it to FAILED_2, if it was it FAILED_2, + move it to FAILED_3. + + In both cases, write what occurred in the job directory in the status.txt + file (append your message, don't remove anything); + +#. Sometimes, while choosing a job, check in the RUNNING directory if any + symlink is not too old (use the name of the link). It may means that a + node crashed while processing this job. Move it to the RUNNING directory + of the next failure level (TODO → FAILED_1 → FAILED_2 → FAILED_3) and try + process it. \ No newline at end of file Modified: trunk/src/site/rst/diswork-fs/history.rst =================================================================== --- trunk/src/site/rst/diswork-fs/history.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-fs/history.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,5 +1,9 @@ +========== +Historique +========== + Premier prototype -================= +----------------- L'objectif était de réaliser un système opportuniste : à chaque fois que des données sont envoyées par un nœud parce qu'un autre nœud les a demandées, Modified: trunk/src/site/rst/diswork-fs/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-08 12:13:22 UTC (rev 66) @@ -1,6 +1,10 @@ +============================= How Diswork File System works ============================= +Diswork File System uses a map to store every data +-------------------------------------------------- + Diswork File System uses a map to store data persistently. We illustrate the use of a file system by setting the content of "/", @@ -122,4 +126,11 @@ When reading and writing in storage, split is done transparently. When reading, a Stream is returned: it loads data blocks after blocks when needed inner class Storage.SplitBlocksInputStream}). -When writing, data are split in blocks of a maximum configurable size. \ No newline at end of file +When writing, data are split in blocks of a maximum configurable size. + +Diswork File System manage concurrency +-------------------------------------- + +DFS uses copy-on-write to deal with concurrency. Nothing is overridden, when +a file is replaced, the new file is written on the FS without removing the +old file. The old file data is kept and actually removed later. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-07 16:24:46 UTC (rev 65) +++ trunk/src/site/site_en.xml 2010-06-08 12:13:22 UTC (rev 66) @@ -13,26 +13,26 @@ </poweredBy> <body> - + <breadcrumbs> <item name="${project.name}" href="index.html" /> </breadcrumbs> - + <menu name="Users"> <item href="user/install.html" name="Install"/> <item href="user/starting.html" name="Starting"/> </menu> - + <menu name="Developer"> <item href="devel/draft.html" name="First draft"/> </menu> - + <menu name="diswork daemon"> - <item href="diswork-fs/how_it_works.html" name="How it works"/> + <item href="diswork-daemon/how_it_works.html" name="How it works"/> </menu> - + <menu name="diswork FS"> - <item href="diswork-fs/history.html" name="Install"/> + <item href="diswork-fs/history.html" name="History of the project"/> <item href="diswork-fs/how_it_works.html" name="How it works"/> </menu>

1 0

r65 - in trunk: . diswork-daemon diswork-daemon/src diswork-daemon/src/main diswork-daemon/src/test diswork-fs/src/main/java/org/nuiton/disworkfs/storage src/site src/site/rst src/site/rst/diswork-daemon
by bleny＠users.nuiton.org June 7, 2010

June 7, 2010

Author: bleny Date: 2010-06-07 18:24:46 +0200 (Mon, 07 Jun 2010) New Revision: 65 Url: http://nuiton.org/repositories/revision/diswork/65 Log: creation module diswork daemon, code style Added: trunk/diswork-daemon/ trunk/diswork-daemon/LICENSE.txt trunk/diswork-daemon/README.txt trunk/diswork-daemon/changelog.txt trunk/diswork-daemon/pom.xml trunk/diswork-daemon/src/ trunk/diswork-daemon/src/main/ trunk/diswork-daemon/src/main/java/ trunk/diswork-daemon/src/test/ trunk/diswork-daemon/src/test/java/ trunk/src/site/rst/diswork-daemon/ trunk/src/site/rst/diswork-daemon/how_it_works.rst Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/pom.xml trunk/src/site/site_en.xml Added: trunk/diswork-daemon/LICENSE.txt =================================================================== --- trunk/diswork-daemon/LICENSE.txt (rev 0) +++ trunk/diswork-daemon/LICENSE.txt 2010-06-07 16:24:46 UTC (rev 65) @@ -0,0 +1,166 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + Added: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml (rev 0) +++ trunk/diswork-daemon/pom.xml 2010-06-07 16:24:46 UTC (rev 65) @@ -0,0 +1,49 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.nuiton</groupId> + <artifactId>diswork</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <groupId>org.nuiton.diswork</groupId> + <artifactId>diswork-daemon</artifactId> + + <packaging>jar</packaging> + <name>Diswork daemon</name> + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>diswork-fs</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-utils</artifactId> + </dependency> + +  + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + </build> + +</project> \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-07 15:48:17 UTC (rev 64) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-07 16:24:46 UTC (rev 65) @@ -102,8 +102,10 @@ if (blockIdsIndex < blockIds.length) { // closing previously opened stream IOUtils.closeQuietly(currentBlock); - currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); - log.debug("new current block (size = " + currentBlock.available() + ")"); + currentBlock = new ByteArrayInputStream( + map.get(blockIds[blockIdsIndex])); + log.debug("new current block (size = " + + currentBlock.available() + ")"); } else { return -1; } @@ -128,10 +130,11 @@ String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; - numberOfAvailableBytes = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + numberOfAvailableBytes = + EntryUtil.getTotalSizeFromMetaBlock(metaBlock); log.debug("initializing stream with meta block \"" + metaBlock - + "\" (" + blockIds.length + " blocks, " + numberOfAvailableBytes - + " bytes)"); + + "\" (" + blockIds.length + " blocks, " + + numberOfAvailableBytes + " bytes)"); } } @@ -335,7 +338,8 @@ unLock(key); } else { - throw new ConcurrentModificationException("key " + key + " is locked"); + throw new ConcurrentModificationException("key " + key + + " is locked"); } } @@ -385,7 +389,9 @@ Boolean result = null; // if there was no lock or if current lock is mine - if (lock == null || (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { + if (lock == null || + (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { + // file is not locked, we have took the lock result = true; } else { @@ -430,7 +436,8 @@ if (obsoleteMetaBlock != null) { String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); - log.info("removing " + obsoleteBlocksIds.length + " old blocks"); + log.info("removing " + obsoleteBlocksIds.length + + " old blocks"); for (String obsoleteBlockId : obsoleteBlocksIds) { removeKey(obsoleteBlockId); } Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-07 15:48:17 UTC (rev 64) +++ trunk/pom.xml 2010-06-07 16:24:46 UTC (rev 65) @@ -19,6 +19,7 @@ <modules> <module>diswork-fs</module> + <module>diswork-daemon</module> </modules> <dependencyManagement> Added: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst (rev 0) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-07 16:24:46 UTC (rev 65) @@ -0,0 +1,4 @@ +How Diswork Daemon works +======================== + +Diswork Daemon uses Diswork File System. \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-07 15:48:17 UTC (rev 64) +++ trunk/src/site/site_en.xml 2010-06-07 16:24:46 UTC (rev 65) @@ -27,6 +27,10 @@ <item href="devel/draft.html" name="First draft"/> </menu> + <menu name="diswork daemon"> + <item href="diswork-fs/how_it_works.html" name="How it works"/> + </menu> + <menu name="diswork FS"> <item href="diswork-fs/history.html" name="Install"/> <item href="diswork-fs/how_it_works.html" name="How it works"/>

1 0

r64 - in trunk: . diswork-fs diswork-fs/src/main/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org June 7, 2010

June 7, 2010

Author: bleny Date: 2010-06-07 17:48:17 +0200 (Mon, 07 Jun 2010) New Revision: 64 Url: http://nuiton.org/repositories/revision/diswork/64 Log: maj pom Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-07 14:19:26 UTC (rev 63) +++ trunk/diswork-fs/pom.xml 2010-06-07 15:48:17 UTC (rev 64) @@ -38,11 +38,11 @@ <artifactId>PeerUnit</artifactId> </dependency> <dependency> - <groupId>org.planx.xmlstore</groupId> - <artifactId>xmlstore</artifactId> + <groupId>org.planx</groupId> + <artifactId>koala-xmlstore</artifactId> </dependency> - +  <dependency> <groupId>junit</groupId> @@ -50,7 +50,7 @@ <scope>test</scope> </dependency> </dependencies> - + <build> <plugins> <plugin> @@ -65,5 +65,5 @@ </plugin> </plugins> </build> - + </project> \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-07 14:19:26 UTC (rev 63) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-07 15:48:17 UTC (rev 64) @@ -29,6 +29,8 @@ import java.net.Socket; import java.net.UnknownHostException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; /** @@ -59,6 +61,9 @@ */ public class DisworkFileSystemConfig extends ApplicationConfig { + private static final Log log = + LogFactory.getLog(DisworkFileSystemConfig.class); + protected static Integer port = 19000; /** @@ -84,7 +89,7 @@ result = temp.getLocalAddress(); temp.close(); } catch (IOException e) { - // TODO 20100602 bleny do something ? + log.warn("can't get external IP address"); } } return result.getHostAddress(); @@ -100,11 +105,6 @@ public int getBlockSize() { return getOptionAsInt("diswork.fs.blocks_size"); } - - @Deprecated - public boolean useInMemoryMap() { - return "inmemory".equals(getOption("diswork.fs.map_type")); - } public Integer getUsedPort() { return getOptionAsInt("diswork.fs.use_port"); Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-07 14:19:26 UTC (rev 63) +++ trunk/pom.xml 2010-06-07 15:48:17 UTC (rev 64) @@ -79,8 +79,8 @@ <version>2.1</version> </dependency> <dependency> - <groupId>org.planx.xmlstore</groupId> - <artifactId>xmlstore</artifactId> + <groupId>org.planx</groupId> + <artifactId>koala-xmlstore</artifactId> <version>0.4.12</version> </dependency> <dependency>

1 0

r63 - in trunk: . diswork-fs diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit diswork-fs/src/main/java/org/nuiton/disworkfs/storage diswork-fs/src/main/resources diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org June 7, 2010

June 7, 2010

Author: bleny Date: 2010-06-07 16:19:26 +0200 (Mon, 07 Jun 2010) New Revision: 63 Url: http://nuiton.org/repositories/revision/diswork/63 Log: doc, commentaires, headers, pom (negligeance des tests impliquant Pastry) Added: trunk/diswork-fs/run_demo.sh trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java trunk/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/pom.xml 2010-06-07 14:19:26 UTC (rev 63) @@ -50,4 +50,20 @@ <scope>test</scope> </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> +  + <excludes> + <exclude>**/*Pastry*.java</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> + </project> \ No newline at end of file Added: trunk/diswork-fs/run_demo.sh =================================================================== --- trunk/diswork-fs/run_demo.sh (rev 0) +++ trunk/diswork-fs/run_demo.sh 2010-06-07 14:19:26 UTC (rev 63) @@ -0,0 +1,2 @@ +mvn -e exec:java -Dexec.mainClass=org.nuiton.disworkfs.Demo -Dexec.args="consumer 9001" -Dexec.classpathScope=test +mvn -e exec:java -Dexec.mainClass=org.nuiton.disworkfs.Demo -Dexec.args="producer 9002 127.0.0.1 9001" -Dexec.classpathScope=test Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-07 14:19:26 UTC (rev 63) @@ -37,6 +37,11 @@ private static final Log log = LogFactory.getLog(Demo.class); + protected static final String USAGE = + "Usage :\n" + + "org.nuiton.disworkfs.Demo producer|consumer" + + " usePort [bootStrapIp bootStrapPort]"; + protected static DisworkFileSystem fileSystem; protected static class Producer implements Runnable { @@ -47,66 +52,42 @@ @Override public void run() { while (true) { - try { if (fileSystem.exists("/todo")) { - if (waitingForJob == null) { - Random random = new Random(); - waitingForJob = "/todo/job-" + random.nextInt(); Integer randomInteger = random.nextInt(); expectedResult = randomInteger + 1; String job = randomInteger.toString(); - InputStream source = IOUtils.toInputStream(job); - - - log.info("writing job " + waitingForJob + " (" + randomInteger + ")"); - + log.info("writing job " + waitingForJob + " (" + + randomInteger + ")"); fileSystem.write(waitingForJob, source); - source.close(); - - List<String> todos = fileSystem.readDirectory("/todo"); - - System.out.println("number of jobs : " + todos.size()); - } else { String resultPath = waitingForJob + ".result"; if (fileSystem.exists(resultPath)) { log.info("results has been published"); - - InputStream resultStream = fileSystem.read(resultPath); - - String resultString = IOUtils.toString(resultStream); - + InputStream resultStream = + fileSystem.read(resultPath); + String resultString = + IOUtils.toString(resultStream); Integer result = Integer.parseInt(resultString); - - log.info("result is " + result + "(expected : " + expectedResult + ")"); - - System.out.println("result is " + result + "(expected : " + expectedResult + ")"); - + log.info("result is " + result + " (expected : " + + expectedResult + ")"); fileSystem.delete(resultPath); fileSystem.delete(waitingForJob); - waitingForJob = null; expectedResult = null; - } - } - - } else { fileSystem.createDirectory("/todo"); } - } catch (IOException e) { log.error(e); } - try { Thread.sleep(15 * 1000); } catch (InterruptedException e) { @@ -121,90 +102,86 @@ @Override public void run() { while (true) { - try { if (fileSystem.exists("/todo")) { - List<String> todos = fileSystem.readDirectory("/todo"); - - System.out.println("todos " + todos.size() + " jobs"); - - if (todos.isEmpty()) { log.info("nothing to do"); } else { // taking a random job Random random = new Random(); - String todo = todos.get(random.nextInt(todos.size())); - + String todo = todos.get( + random.nextInt(todos.size()) + ); if (!todo.endsWith(".result")) { String jobPath = "/todo/" + todo; log.info("reading the job " + jobPath); - InputStream in = fileSystem.read(jobPath); - String operation = IOUtils.toString(in); - // in.close(); - log.info("operation to do " + operation); - Integer i = Integer.parseInt(operation); - i += 1; - String result = i.toString(); - log.info("result is " + result); - - InputStream source = IOUtils.toInputStream(result); - + InputStream source = + IOUtils.toInputStream(result); fileSystem.write(jobPath + ".result", source); - source.close(); - } } - } else { fileSystem.createDirectory("/todo"); } - } catch (IOException e) { log.error(e); } - try { Thread.sleep(10 * 1000); } catch (InterruptedException e) { log.info("sleep interrupted", e); } - } } } public static void printUsage() { - System.out.println( - "Usage :\n" - + "org.nuiton.disworkfs.Demo producer|consumer usePort [bootStrapIp bootStrapPort]" - ); + System.out.println(USAGE); } /** + * + * <dl> + * <dt>producer|consumer (required)</dt> + * <dd>a producer will produce operations todo, a consumer may read + * operations and try to execute them + * </dd> + * <dt>port (required)</dt> + * <dd>the port to use</dd> + * <dt>bootstrapip (optional)</dt> + * <dd>if not provided, the node will bootstrap itself, creating a + * new network. If provided, the node will first try to join a + * network</dd> + * <dt>bootstrap port (required if bootstrapip is set)</dt> + * <dd>the port used by the node to join</dd> + * </dl> + * + * Example : + * org.nuiton.disworkfs.Demo consumer 9001 + * org.nuiton.disworkfs.Demo producer 9002 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9003 127.0.0.1 9002 + * org.nuiton.disworkfs.Demo producer 9004 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9005 127.0.0.1 9003 + * * @param args * @throws IOException */ public static void main(String[] args) throws IOException { - if (args.length == 2) { - - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); - fileSystem = new DisworkFileSystem(config); - if ("producer".equals(args[0])) { Thread t = new Thread(new Producer()); log.info("starting a producer"); @@ -214,17 +191,13 @@ log.info("starting a consumer"); t.start(); } - } else if (args.length == 4) { - - DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); - + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); config.setOption("diswork.fs.bootstrap.ip", args[2]); config.setOption("diswork.fs.bootstrap.port", args[3]); - fileSystem = new DisworkFileSystem(config); - if ("producer".equals(args[0])) { Thread t = new Thread(new Producer()); log.info("starting a producer"); @@ -234,10 +207,8 @@ log.info("starting a consumer"); t.start(); } - } else { printUsage(); } } - -} +} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-07 14:19:26 UTC (rev 63) @@ -572,29 +572,26 @@ // FIXME 20105021 bleny works fine but is not understandable String result = null; + // if path is "/", recursion can be initiated with this value : + // (it returns "/" as the id where to get the content of "/" if (path.equals(EntryUtil.ROOT_DIRECTORY)) { return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + EntryUtil.ROOT_DIRECTORY; } - + String parentPath = EntryUtil.getParentFromPath(path); - + if (content == null) { - // start of walk + // start the recursion from root directory content = storage.getRootDirectory(); result = walk(path, EntryUtil.ROOT_DIRECTORY, content); } else if (parentPath.equals(current)) { - // in the last directory - // recuperation de l'element a traiter dans p + // we are now in the last directory + // ie if path is a/b/p we are at a/b + // p is the name of the element in a/b we search for String tail = path.substring(current.length()); - - /* - log.info("tail " + tail); - String[] elementsNames = tail.split("/"); - String p = elementsNames[0]; - */ String p = EntryUtil.getNameFromPath(tail); log.info("in final dir " + current + ", looking for " + p); @@ -603,9 +600,12 @@ result = entry; } else { // in middle of path - + // if path is a/b/c/d/e/f, we may be at b, c, d... + String tail; // the path still remaining when in current + // ie if we are in c, tail is "d/e/f" + // if we are at root directory, deal with the "/" if (current.equals(EntryUtil.ROOT_DIRECTORY)) { tail = path.substring(current.length()); } else { @@ -619,7 +619,7 @@ log.info("in intermediate dir " + current + ", looking for " + p); - // mise a jour de current + // updating current for recursion if (current.equals(EntryUtil.ROOT_DIRECTORY)) { current = ""; // avoid "//path" next line } @@ -629,6 +629,8 @@ if (entry == null) { result = null; } else { + // we have found the entry to call recursion + if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getIdFromEntry(entry); content = storage.getDirectory(id); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-07 14:19:26 UTC (rev 63) @@ -94,11 +94,7 @@ setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB setDefaultOption("diswork.fs.map_type", "inmemory"); - setDefaultOption("diswork.fs.use_port", "9001"); - /* - setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); - setDefaultOption("diswork.fs.bootstrap.port", "9001"); - */ + setDefaultOption("diswork.fs.use_port", port.toString()); } public int getBlockSize() { Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java 2010-06-07 14:19:26 UTC (rev 63) @@ -0,0 +1,29 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +/** + * In this package can be found some Test classes for PeerUnit framework + */ + +package org.nuiton.disworkfs.peerunit; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-07 14:19:26 UTC (rev 63) @@ -27,4 +27,4 @@ import java.io.Closeable; import java.util.Map; -public interface DisworkMap extends Map<String, byte[]>, Closeable {} +public interface DisworkMap extends Map<String, byte[]>, Closeable {} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-07 14:19:26 UTC (rev 63) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.DataInput; @@ -4,7 +28,6 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import java.math.BigInteger; import java.net.InetSocketAddress; import java.util.Collection; import java.util.Map; @@ -17,14 +40,13 @@ import org.nuiton.disworkfs.DisworkFileSystemConfig; import org.planx.xmlstore.routing.Identifier; import org.planx.xmlstore.routing.Kademlia; -import org.planx.xmlstore.routing.RoutingException; public class KademliaDisworkMap implements DisworkMap { private static final Log log = LogFactory.getLog(KademliaDisworkMap.class); protected Kademlia kad; - + public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); @@ -43,8 +65,6 @@ UUID uuid = UUID.nameUUIDFromBytes(EntryUtil.stringToBytes(s)); s = uuid.toString(); - - /* byte[] bytes = EntryUtil.stringToBytes(s); Identifier id = null; @@ -108,7 +128,6 @@ byte[] previousValue = null; try { previousValue = get(key); - Identifier id = stringToIdentifier((String) key); kad.put(id, value); } catch (Exception e) { @@ -124,7 +143,6 @@ byte[] previousValue = null; try { previousValue = get(key); - Identifier id = stringToIdentifier((String) key); kad.remove(id); } catch (Exception e) { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-07 14:19:26 UTC (rev 63) @@ -80,7 +80,8 @@ } } - protected abstract class MyContinuation<R, E extends Exception> implements Continuation<R, Exception> { + protected abstract class MyContinuation<R, E extends Exception> + implements Continuation<R, Exception> { protected boolean finished = false; @@ -184,7 +185,8 @@ // abort if can't join if (node.joinFailed()) { - throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason()); + throw new IOException("Could not join the FreePastry ring. " + + "Reason : " + node.joinFailedReason()); } } } @@ -216,7 +218,8 @@ } - protected class ContainsKeyContinuation extends MyContinuation<Boolean, Exception> { + protected class ContainsKeyContinuation extends + MyContinuation<Boolean, Exception> { public Boolean result = null; @@ -231,15 +234,16 @@ @Override public boolean containsKey(Object key) { Id id = pastryIdFactory.buildId((String) key); - ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation(); - past.existsInOverlay(id, containsKeyContinuation); - containsKeyContinuation.waitUntilFinished(); - boolean result = containsKeyContinuation.result; + ContainsKeyContinuation continuation = new ContainsKeyContinuation(); + past.existsInOverlay(id, continuation); + continuation.waitUntilFinished(); + boolean result = continuation.result; log.info("containsKey " + key + " (id = " + id + ") returns " + result); return result; } - protected class GetContinuation extends MyContinuation<PastContent, Exception> { + protected class GetContinuation extends + MyContinuation<PastContent, Exception> { public byte[] result = null; @@ -278,7 +282,8 @@ return result; } - protected class PutContinuation extends MyContinuation<Boolean[], Exception> { + protected class PutContinuation extends + MyContinuation<Boolean[], Exception> { @Override public void receiveResult(Boolean[] result) { log.info("insert result received : " + Arrays.toString(result)); @@ -311,7 +316,8 @@ return previousValue; } - protected class RemoveContinuation extends MyContinuation<Boolean, Exception> { + protected class RemoveContinuation extends + MyContinuation<Boolean, Exception> { public Boolean result = null; @@ -334,7 +340,8 @@ RemoveContinuation removeContinuation = new RemoveContinuation(); past.remove(id, removeContinuation); removeContinuation.waitUntilFinished(); - log.info("atomic remove " + key + " has returned " + removeContinuation.result); + log.info("atomic remove " + key + " has returned " + + removeContinuation.result); } @Override @@ -344,7 +351,8 @@ log.info("remove " + key + " returns null"); } else { atomicRemove(key); - log.info("remove " + key + " returns " + previousValue.length + " bytes"); + log.info("remove " + key + " returns " + previousValue.length + + " bytes"); } return previousValue; } Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-06-07 14:19:26 UTC (rev 63) @@ -5,5 +5,5 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=INFO +log4j.logger.org.nuiton.disworkfs=WARN log4j.logger.org.nuiton.disworkfs.Demo=INFO \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -2,7 +2,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -10,8 +9,6 @@ import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.Before; import org.junit.Test; Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -2,7 +2,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -10,8 +9,6 @@ import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.Before; import org.junit.Test; @@ -32,7 +29,7 @@ } - /* + @Test public void testMultipleNodes1() throws Exception { DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); @@ -61,19 +58,18 @@ assertArrayEquals(bytes, getResult); } - /* + @Test public void testMultipleNodes3() throws Exception { fileSystem.createDirectory("/mydir"); try { - log.info("----------------------trying to create second directory"); fileSystem.createDirectory("/myseconddir"); } catch (ConcurrentModificationException e) { fail(); } } - */ + } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -28,12 +28,10 @@ @After public void tearDown() throws Exception { - /* if (map1 != null) map1.close(); if (map2 != null) map2.close(); - */ } /** Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -1,13 +1,17 @@ package org.nuiton.disworkfs.storage; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.junit.Before; import org.junit.Test; import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.planx.xmlstore.routing.Identifier; public class KademliaDisworkMapTest extends AbstractDisworkMapTest { @@ -20,12 +24,42 @@ map2 = new KademliaDisworkMap(config2); } + /** + * this tests the key generation algorithm, it has different properties to + * check : + * * two call with same param should return same key + * * two call with different parameter should return differents keys + * * a call should never return null + */ @Test public void testStringToIndentifier() { - assertNotNull(KademliaDisworkMap.stringToIdentifier("/")); - assertNotNull(KademliaDisworkMap.stringToIdentifier( - "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" - /* + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" - + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir"*/)); + + List<String> paths = new ArrayList<String>(); + + // a set will remove doubloons + Set<Identifier> keys = new HashSet<Identifier>(); + + paths.add("/"); + paths.add("/dir"); + paths.add("/dir/dir"); + paths.add("/a_directory_with_a_long_name_may_lead_to_the_raise_of_an_" + + "exception"); + paths.add("1111111111111111111111111111111111111111111111111111111111"); + paths.add("1111111111111111111111111111111111111111111111111111111112"); + + for (String path : paths) { + Identifier key = KademliaDisworkMap.stringToIdentifier(path); + assertNotNull(key); + + // keys should be equals (depend only from path, not random factor) + Identifier otherKey = KademliaDisworkMap.stringToIdentifier(path); + assertEquals(key, otherKey); + + keys.add(key); + } + + // check differents paths have different keys + // if keys set is too small, there are collisions + assertEquals(paths.size(), keys.size()); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-07 14:19:26 UTC (rev 63) @@ -1,17 +1,17 @@ -package org.nuiton.disworkfs.storage; - -import org.junit.Before; -import org.nuiton.disworkfs.DisworkFileSystemConfig; - -public class PastryDisworkMapTest extends AbstractDisworkMapTest { - - @Before - public void setUp() throws Exception { - DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newPastryDisworkConfig(); - map1 = new PastryDisworkMap(config1); - - DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newPastryDisworkConfig(config1.getUsedPort()); - map2 = new PastryDisworkMap(config2); - } - -} +//package org.nuiton.disworkfs.storage; +// +//import org.junit.Before; +//import org.nuiton.disworkfs.DisworkFileSystemConfig; +// +//public class PastryDisworkMapTest extends AbstractDisworkMapTest { +// +// @Before +// public void setUp() throws Exception { +// DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newPastryDisworkConfig(); +// map1 = new PastryDisworkMap(config1); +// +// DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newPastryDisworkConfig(config1.getUsedPort()); +// map2 = new PastryDisworkMap(config2); +// } +// +//} Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-04 18:15:37 UTC (rev 62) +++ trunk/pom.xml 2010-06-07 14:19:26 UTC (rev 63) @@ -8,11 +8,11 @@   - <parent> - <groupId>org.nuiton</groupId> - <artifactId>mavenpom4redmine</artifactId> - <version>2.1.5</version> - </parent> + <parent> + <groupId>org.nuiton</groupId> + <artifactId>mavenpom4redmine</artifactId> + <version>2.1.5</version> + </parent> <artifactId>diswork</artifactId> <version>0.0.1-SNAPSHOT</version> @@ -79,11 +79,16 @@ <version>2.1</version> </dependency> <dependency> - <groupId>org.planx.xmlstore</groupId> - <artifactId>xmlstore</artifactId> - <version>0.4.12</version> + <groupId>org.planx.xmlstore</groupId> + <artifactId>xmlstore</artifactId> + <version>0.4.12</version> </dependency> - + <dependency> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + <version>2.0</version> + </dependency> +  <dependency> <groupId>junit</groupId>

1 0

r62 - in trunk: . diswork-fs diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit diswork-fs/src/main/java/org/nuiton/disworkfs/storage diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org June 4, 2010

June 4, 2010

Author: bleny Date: 2010-06-04 20:15:37 +0200 (Fri, 04 Jun 2010) New Revision: 62 Url: http://nuiton.org/repositories/revision/diswork/62 Log: implementation kademlia, tests, demo, fabriques de configs, remaniement des tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java Removed: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java Modified: trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java trunk/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/pom.xml 2010-06-04 18:15:37 UTC (rev 62) @@ -37,6 +37,11 @@ <groupId>fr.inria.peerunit</groupId> <artifactId>PeerUnit</artifactId> </dependency> + <dependency> + <groupId>org.planx.xmlstore</groupId> + <artifactId>xmlstore</artifactId> + </dependency> +  <dependency> Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-04 18:15:37 UTC (rev 62) @@ -53,7 +53,6 @@ if (waitingForJob == null) { - // taking a random job Random random = new Random(); waitingForJob = "/todo/job-" + random.nextInt(); @@ -87,6 +86,8 @@ log.info("result is " + result + "(expected : " + expectedResult + ")"); + System.out.println("result is " + result + "(expected : " + expectedResult + ")"); + fileSystem.delete(resultPath); fileSystem.delete(waitingForJob); @@ -126,6 +127,9 @@ List<String> todos = fileSystem.readDirectory("/todo"); + System.out.println("todos " + todos.size() + " jobs"); + + if (todos.isEmpty()) { log.info("nothing to do"); } else { @@ -195,7 +199,7 @@ if (args.length == 2) { - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); @@ -213,7 +217,7 @@ } else if (args.length == 4) { - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); + DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); config.setOption("diswork.fs.use_port", args[1]); config.setOption("diswork.fs.bootstrap.ip", args[2]); @@ -221,11 +225,11 @@ fileSystem = new DisworkFileSystem(config); - if ("producer".equals(args[2])) { + if ("producer".equals(args[0])) { Thread t = new Thread(new Producer()); log.info("starting a producer"); t.start(); - } else if ("consumer".equals(args[2])) { + } else if ("consumer".equals(args[0])) { Thread t = new Thread(new Consumer()); log.info("starting a consumer"); t.start(); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-04 18:15:37 UTC (rev 62) @@ -34,6 +34,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.storage.DisworkMap; import org.nuiton.disworkfs.storage.EntryUtil; import org.nuiton.disworkfs.storage.Storage; @@ -657,5 +658,9 @@ public void close() throws IOException { storage.close(); } + + protected void setMap(DisworkMap map) { + storage.setMap(map); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-04 18:15:37 UTC (rev 62) @@ -24,6 +24,11 @@ */ package org.nuiton.disworkfs; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + import org.nuiton.util.ApplicationConfig; /** @@ -37,34 +42,153 @@ * (by default, 10 MiB) * </dd> * </dl> + * + * + * This class provides utilities to write tests easily. + * + * You can get multiples diswork configs ready to use. All the instances uses + * a new port and the local IP. + * <pre> + * c = newDisworkConfig(); // create a config for a bootstrap node + * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that + * // will bootstrap by joining the + * // first node + * c3 = newDisworkConfig(c.getUsedPort()) + * </pre> + * */ public class DisworkFileSystemConfig extends ApplicationConfig { + protected static Integer port = 19000; + + /** + * returns a new port, returned value change at each call. + * @return + */ + public static Integer getPort() { + port += 1; + return port; + } + + /** + * returns the IP on the local machine. Trying to get an public IP or a LAN + * IP or the loopback IP if there is no other interface + * @return + * @throws UnknownHostException + */ + public static String getIp() throws UnknownHostException { + InetAddress result = InetAddress.getLocalHost(); + if (result.isLoopbackAddress()) { + try { + Socket temp = new Socket("microsoft.com", 80); + result = temp.getLocalAddress(); + temp.close(); + } catch (IOException e) { + // TODO 20100602 bleny do something ? + } + } + return result.getHostAddress(); + } + public DisworkFileSystemConfig() { setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB - setDefaultOption("diswork.fs.use_in_memory_map", "false"); + + setDefaultOption("diswork.fs.map_type", "inmemory"); setDefaultOption("diswork.fs.use_port", "9001"); - setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); + /* + setDefaultOption("diswork.fs.bootstrap.ip", "192.168.99.119"); setDefaultOption("diswork.fs.bootstrap.port", "9001"); + */ } public int getBlockSize() { return getOptionAsInt("diswork.fs.blocks_size"); } + @Deprecated public boolean useInMemoryMap() { - return getOptionAsBoolean("diswork.fs.use_in_memory_map"); + return "inmemory".equals(getOption("diswork.fs.map_type")); } public Integer getUsedPort() { return getOptionAsInt("diswork.fs.use_port"); } + public void setUsedPort(Integer port) { + setOption("diswork.fs.use_port", port.toString()); + } + public String getBootstrapIp() { return getOption("diswork.fs.bootstrap.ip"); } + public void setBootstrapIp(String ip) { + setOption("diswork.fs.bootstrap.ip", ip); + } + public Integer getBootstrapPort() { return getOptionAsInt("diswork.fs.bootstrap.port"); } + + public void setBootstrapPort(Integer port) { + setOption("diswork.fs.bootstrap.port", port.toString()); + } + + public String getMapType() { + return getOption("diswork.fs.map_type"); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a single-node instance of DisworkFS + * @return + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig() + throws UnknownHostException { + return newPastryDisworkConfig(null); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a multiple-node instance of DisworkFS on a same machine. + * @param bootstrapPort the port on the same machine where another node can + * be found to bootstrap + * @return a complete config + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "pastry"); + result.setOption("diswork.fs.use_port", port); + result.setOption("diswork.fs.bootstrap.ip", ip); + if (bootstrapPort == null) { + result.setOption("diswork.fs.bootstrap.port", port); + } else { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + } + return result; + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig() + throws UnknownHostException { + return newKademliaDisworkConfig(null); + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "kademlia"); + result.setOption("diswork.fs.use_port", port); + if (bootstrapPort != null) { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + result.setOption("diswork.fs.bootstrap.ip", ip); + } + return result; + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -34,6 +34,8 @@ import java.util.Arrays; import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.DisworkFileSystem; import org.nuiton.disworkfs.DisworkFileSystemConfig; @@ -42,28 +44,53 @@ public class DisworkFileSystemTest extends TestCaseImpl { - protected Integer port = 9001; + private static final Log log = + LogFactory.getLog(DisworkFileSystemTest.class); + // protected Integer port = 31000; + protected DisworkFileSystem fileSystem; protected byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; @TestStep(range="0",timeout=1000000, order = 0) public void testConnect() throws Exception { - Integer myPort = port + getId(); - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); - config.setOption("diswork.fs.use_port", myPort.toString()); + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setUsedPort(31000); fileSystem = new DisworkFileSystem(config); + + config.printConfig(); + + log.info("i bootstrap : " + DisworkFileSystemConfig.getIp() + ":" + 31000); + put(0, DisworkFileSystemConfig.getIp()); + put(1, 31000); + } @TestStep(range="1",timeout=1000000, order = 1) public void testConnect1() throws Exception { - Integer myPort = port + getId(); - DisworkFileSystemConfig config = new DisworkFileSystemConfig(); - config.setOption("diswork.fs.use_port", myPort.toString()); + // Integer myPort = port + getId(); + Integer myPort = 31001; + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setUsedPort(myPort); + + // get bootstrap info + + String bootstrapIp = (String) get(0); + Integer bootstrapPort = (Integer) get(1); + + config.setBootstrapIp("127.0.0.1"); + config.setBootstrapPort(bootstrapPort); + + config.printConfig(); + + log.info("bootstrap is " + bootstrapIp + ":" + bootstrapPort); + fileSystem = new DisworkFileSystem(config); } - /* + @TestStep(range="0",timeout=1000000, order = 2) public void testWrite() throws Exception { InputStream source = new ByteArrayInputStream(bytes); @@ -76,34 +103,28 @@ try { InputStream source = fileSystem.read("/myfile"); byte[] readResult = IOUtils.toByteArray(source); - source.close(); + // source.close(); boolean arraysContentEquals = Arrays.equals(bytes, readResult); + + System.out.println("arraysContentEquals = " + arraysContentEquals); assertTrue(arraysContentEquals); } catch (FileNotFoundException e) { + System.out.println("file not found"); fail(); } } - */ + @TestStep(range="1",timeout=1000000, order = 4) public void testCreateDir() throws Exception { fileSystem.createDirectory("/mydir"); - fileSystem.createDirectory("/myseconddir"); } - @TestStep(range="0-1",timeout=1000000, order = 5) - public void fake() throws Exception { - - } - - /* - @TestStep(range="0-1",timeout=1000000, order = 5) + @TestStep(range="0",timeout=1000000, order = 5) public void testCreateDir2() throws Exception { assertTrue(fileSystem.exists("/mydir")); - assertTrue(fileSystem.exists("/myseconddir")); } - */ - /* - @TestStep(range="1",timeout=1000000, order = 6) + + @TestStep(range="0",timeout=1000000, order = 6) public void testCreateSubDir() throws Exception { fileSystem.createDirectory("/mydir/mysubdir"); } @@ -113,5 +134,5 @@ assertTrue(fileSystem.exists("/mydir/mysubdir")); assertEquals(1, fileSystem.readDirectory("/mydir").size()); } - */ + } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,185 @@ +package org.nuiton.disworkfs.storage; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.planx.xmlstore.routing.Identifier; +import org.planx.xmlstore.routing.Kademlia; +import org.planx.xmlstore.routing.RoutingException; + +public class KademliaDisworkMap implements DisworkMap { + + private static final Log log = LogFactory.getLog(KademliaDisworkMap.class); + + protected Kademlia kad; + + public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { + kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); + + if (config.getBootstrapIp() != null) { + InetSocketAddress bootstrap = new InetSocketAddress( + config.getBootstrapIp(), + config.getBootstrapPort()); + kad.connect(bootstrap); + } + + //Identifier.IDSIZE = 1024; + log.info("using " + Identifier.IDSIZE + " bytes for identifiers"); + } + + protected static Identifier stringToIdentifier(String s) { + UUID uuid = UUID.nameUUIDFromBytes(EntryUtil.stringToBytes(s)); + s = uuid.toString(); + + + + /* + byte[] bytes = EntryUtil.stringToBytes(s); + Identifier id = null; + try { + id = new Identifier(bytes); + } catch (java.lang.IndexOutOfBoundsException e) { + log.error("generated id is out of bound : Identifier.IDSIZE " + + " is too small"); + } + */ + + InputStream in = IOUtils.toInputStream(s); + DataInput dataInput = new DataInputStream(in); + Identifier id = null; + try { + id = new Identifier(dataInput); + } catch (IOException e) { + log.error("unable to create key from string " + s); + } + + + log.info("key for string " + s + " is " + id); + return id; + } + + @Override + public void close() throws IOException { + kad.close(); + } + + @Override + public boolean containsKey(Object key) { + Identifier id = stringToIdentifier((String) key); + Boolean result = null; + try { + result = kad.contains(id); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return result; + } + + @Override + public byte[] get(Object key) { + Identifier id = stringToIdentifier((String) key); + byte[] result = null; + try { + result = (byte[]) kad.get(id); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return result; + } + + @Override + public byte[] put(String key, byte[] value) { + byte[] previousValue = null; + try { + previousValue = get(key); + + Identifier id = stringToIdentifier((String) key); + kad.put(id, value); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return previousValue; + } + + @Override + public byte[] remove(Object key) { + byte[] previousValue = null; + try { + previousValue = get(key); + + Identifier id = stringToIdentifier((String) key); + kad.remove(id); + } catch (Exception e) { + // FIXME 20100604 bleny bad exception management + log.error("Kademlia exception caught", e); + throw new RuntimeException(e); + } + return previousValue; + } + + @Override + @Deprecated + public Collection<byte[]> values() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public void clear() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public boolean containsValue(Object arg0) { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public Set<java.util.Map.Entry<String, byte[]>> entrySet() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public int size() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public void putAll(Map<? extends String, ? extends byte[]> arg0) { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public Set<String> keySet() { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + @Deprecated + public boolean isEmpty() { + throw new UnsupportedOperationException("not yet implemented"); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-04 18:15:37 UTC (rev 62) @@ -147,24 +147,46 @@ protected DisworkFileSystemConfig disworkConfig; - public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { + public Storage(DisworkFileSystemConfig disworkConfig, DisworkMap map) + throws IOException { + this.disworkConfig = disworkConfig; - - // instantiating a map according to config directives - if (disworkConfig.useInMemoryMap()) { - log.info("using in-memory map"); - map = new InMemoryDisworkMap(); + + if (map == null) { + // instantiating a map according to config directives + String mapType = disworkConfig.getOption("diswork.fs.map_type"); + + if (mapType == null) { + log.info("no map type specified"); + this.map = null; + } else { + if ("inmemory".equals(mapType)) { + log.info("using in-memory map"); + this.map = new InMemoryDisworkMap(); + } else if ("pastry".equals(mapType)) { + log.info("using Pastry map"); + this.map = new PastryDisworkMap(disworkConfig); + } else if ("kademlia".equals(mapType)) { + log.info("using Kademlia map"); + this.map = new KademliaDisworkMap(disworkConfig); + } + } } else { - log.info("using Pastry map"); - map = new PastryDisworkMap(disworkConfig); + this.map = map; } + // creating root directory, if needed - if (! map.containsKey(EntryUtil.ROOT_DIRECTORY)) { + if (! this.map.containsKey(EntryUtil.ROOT_DIRECTORY)) { + log.info("creating root directory"); putDirectory(EntryUtil.ROOT_DIRECTORY, EntryUtil.EMPTY_DIRECTORY_CONTENT); } } + + public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { + this(disworkConfig, null); + } /** * @return the content (entries) of the root directory @@ -430,5 +452,9 @@ Long lockAge = currentDate - currentLockTime; return lockAge > LOCK_VALID_TIME; } + + public void setMap(DisworkMap map) { + this.map = map; + } } \ No newline at end of file Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java (from rev 60, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,346 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ConcurrentModificationException; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + + +public abstract class AbstractDisworkFileSystemTest { + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = + System.getProperty("java.io.tmpdir", ".") // temp directory + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 9999; + + static protected DisworkFileSystem fileSystem; + + /** + * This setUp creates in a temp directory a file of size + * {@link randomFileSize} (fulfilling it with random bytes) + * This file can be found at {@link #randomFilePath} + * At the end of the method {@link #fileSystem} is ready to be used + * @throws Exception + */ + @Before + public void setUp() throws Exception { + // create a temp directory for our test + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + // creating random data for the file + Random random = new Random(); + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + /** + * writing a file at root directory should not raise any exception + * @throws Exception + */ + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + /** + * First, write a file then test if exists return true. + * @throws Exception + */ + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte: contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + source.close(); + readResult.close(); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testFailAtWrite() throws Exception { + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + } + + /** + * this use case should raise an exception because writing to a file + * that already exists + */ + @Test(expected = IOException.class) + public void testFailAtDoubleWrite() throws Exception { + InputStream source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + source.close(); + + source = new FileInputStream(randomFilePath); + try { + fileSystem.write("/my_file", source); + } finally { + source.close(); + } + } + + /** + * This test uses mkdir to create dirs and sub-dirs + * @throws Exception + */ + @Test + public void testCreateDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + /** + * Create some folders with mkdir and write files in those directories + * @throws Exception + */ + @Test + public void testWriteInFolder() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", + new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + /** + * create a symbolic link to a file. This test show that we can read the + * file using the link + * @throws Exception + */ + @Test + public void testLinking() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + /** + * Trying to create a link to a wrong target, this sould raise an exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtLinking() throws Exception { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + } + + /** + * Trying to remove files and directories + * @throws Exception + */ + @Test + public void testRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.delete("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.delete("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + /** + * By trying to remove a non-empty directory, this test should raise an + * exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.delete("/my_folder"); + } + + /** + * This tests uses ls + */ + @Test + public void testListDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.createDirectory("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); + + List<String> lsResult = fileSystem.readDirectory("/my_folder"); + + // checking that result contains all the required data + assertTrue(lsResult.contains("my_sub_dir")); + assertTrue(lsResult.contains("my_file")); + assertTrue(lsResult.contains("my_link")); + + // ... and only those + assertEquals(3, lsResult.size()); + + lsResult = fileSystem.readDirectory("/"); + assertEquals(1, lsResult.size()); + } + + @Test + public void testConcurrency() throws Exception { + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + } + + @Test + public void testLinks() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/otherdir"); + + fileSystem.createSymbolicLink("/link", "dir/subdir"); + fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); + List<String> readResult = fileSystem.readDirectory("/dir/subdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subsubdirlink")); + + fileSystem.createDirectory("/link/subsubdirlink/finaldir"); + readResult = fileSystem.readDirectory("/otherdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("finaldir")); + + } +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -2,17 +2,17 @@ import org.junit.Before; -public class DisworkFileSystemInMemoryTest extends DisworkFileSystemTest { +public class DisworkFileSystemInMemoryTest extends AbstractDisworkFileSystemTest { /** - * this code executed after {@link DisworkFileSystemTest#setUp()} + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} * @throws Exception */ @Before public void setUpFileSystem() throws Exception { // finally, initiate the fileSystem DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); - disworkConfig.setOption("diswork.fs.use_in_memory_map", "true"); + disworkConfig.setOption("diswork.fs.map_type", "inmemory"); fileSystem = new DisworkFileSystem(disworkConfig); } Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,80 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Before; +import org.junit.Test; + +public class DisworkFileSystemKademliaTest extends AbstractDisworkFileSystemTest { + + protected Integer bootstrapPort; + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + } + + + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -15,18 +15,18 @@ import org.junit.Before; import org.junit.Test; -public class DisworkFileSystemPastryTest extends DisworkFileSystemTest { +public class DisworkFileSystemPastryTest extends AbstractDisworkFileSystemTest { protected Integer bootstrapPort; /** - * this code executed after {@link DisworkFileSystemTest#setUp()} + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} * @throws Exception */ @Before public void setUpFileSystem() throws Exception { // finally, initiate the fileSystem - DisworkFileSystemConfig disworkConfig1 = Util.newDisworkConfig(); + DisworkFileSystemConfig disworkConfig1 = DisworkFileSystemConfig.newPastryDisworkConfig(); bootstrapPort = disworkConfig1.getUsedPort(); fileSystem = new DisworkFileSystem(disworkConfig1); @@ -35,7 +35,7 @@ /* @Test public void testMultipleNodes1() throws Exception { - DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); assertTrue(fileSystem.exists("/")); @@ -44,7 +44,7 @@ @Test public void testMultipleNodes2() throws Exception { - DisworkFileSystemConfig disworkConfig = Util.newDisworkConfig(bootstrapPort); + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -61,7 +61,7 @@ assertArrayEquals(bytes, getResult); } - + /* @Test public void testMultipleNodes3() throws Exception { Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,346 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ConcurrentModificationException; -import java.util.List; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.util.FileUtil; - - -public abstract class DisworkFileSystemTest { - - /** - * a place to store files for the test it's a subdirectory of the OS temp - * dir e.g. under linux /tmp/disworkfs/tests/ - */ - static protected String tempDirectoryPath = - System.getProperty("java.io.tmpdir", ".") // temp directory - + "/disworkfs/tests"; - - /** - * We will create a file at this path for test purpose - */ - static protected String randomFilePath = tempDirectoryPath + "/randomfile"; - - /** - * The file will have this fixed size - */ - static protected int randomFileSize = 9999; - - static protected DisworkFileSystem fileSystem; - - /** - * This setUp creates in a temp directory a file of size - * {@link randomFileSize} (fulfilling it with random bytes) - * This file can be found at {@link #randomFilePath} - * At the end of the method {@link #fileSystem} is ready to be used - * @throws Exception - */ - @Before - public void setUp() throws Exception { - // create a temp directory for our test - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - // creating random data for the file - Random random = new Random(); - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - - } - - @After - public void tearDown() throws Exception { - // cleaning - FileUtil.deleteRecursively(tempDirectoryPath); - } - - /** - * writing a file at root directory should not raise any exception - * @throws Exception - */ - @Test - public void testWrite() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - } - - /** - * First, write a file then test if exists return true. - * @throws Exception - */ - @Test - public void testExists() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); - } - - /** - * try to read a file that as never been created nor written - */ - @Test(expected = FileNotFoundException.class) - public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); - } - - /** - * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} - * by storing a "-1" byte, can be buggy due to the use of read() - * @throws IOException - */ - @Test - public void testSplit() throws IOException { - - byte[] bytes = new byte[1]; - bytes[0] = -0x1; - - InputStream source; - - source = new ByteArrayInputStream(bytes); - fileSystem.write("/", "my_file", source); - - source.close(); - - - source = new ByteArrayInputStream(bytes); - InputStream readResult = fileSystem.read("/my_file"); - - int read = 0; - byte[] b = new byte[1]; - - read = readResult.read(b); - - assertEquals(1, read); - assertArrayEquals(bytes, b); - - } - - /** - * writing a file at the root directory and reading it. - * finally, compare original source and read result - * byte-to-byte: contents should be equals - * @throws Exception - */ - @Test - public void testWriteRead() throws Exception { - - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/", "my_file", source); - - source.close(); - - InputStream readResult; - - // now, the checks. We read the original file and the result of - // a read() and then compare it byte-to-byte - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - - assertEquals(randomFileSize, source.available()); - assertEquals(randomFileSize, readResult.available()); - - byte[] sourceAsBytes = IOUtils.toByteArray(source); - byte[] readResultAsBytes = IOUtils.toByteArray(readResult); - - assertArrayEquals(sourceAsBytes, readResultAsBytes); - - source.close(); - readResult.close(); - - } - - /** - * this use case should raise an exception because my_folder - * doesn't exists - */ - @Test(expected = IOException.class) - public void testFailAtWrite() throws Exception { - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - } - - /** - * this use case should raise an exception because writing to a file - * that already exists - */ - @Test(expected = IOException.class) - public void testFailAtDoubleWrite() throws Exception { - InputStream source = new FileInputStream(randomFilePath); - fileSystem.write("/my_file", source); - source.close(); - - source = new FileInputStream(randomFilePath); - try { - fileSystem.write("/my_file", source); - } finally { - source.close(); - } - } - - /** - * This test uses mkdir to create dirs and sub-dirs - * @throws Exception - */ - @Test - public void testCreateDirectory() throws Exception { - fileSystem.createDirectory("/my_folder"); - assertTrue(fileSystem.exists("/my_folder")); - fileSystem.createDirectory("/my_folder/my_sub_folder"); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); - } - - /** - * Create some folders with mkdir and write files in those directories - * @throws Exception - */ - @Test - public void testWriteInFolder() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.createDirectory("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder", "my_file", - new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); - } - - /** - * create a symbolic link to a file. This test show that we can read the - * file using the link - * @throws Exception - */ - @Test - public void testLinking() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); - - InputStream source = new FileInputStream(randomFilePath); - InputStream readResult = fileSystem.read("/my_link"); - - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); - } - - /** - * Trying to create a link to a wrong target, this sould raise an exception - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFailAtLinking() throws Exception { - fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); - } - - /** - * Trying to remove files and directories - * @throws Exception - */ - @Test - public void testRemove() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.delete("/my_folder/my_file"); - assertTrue(fileSystem.exists("/my_folder")); - assertFalse(fileSystem.exists("/my_folder/my_file")); - fileSystem.delete("/my_folder"); - assertFalse(fileSystem.exists("/my_folder")); - } - - /** - * By trying to remove a non-empty directory, this test should raise an - * exception - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFailAtRemove() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - - // trying to remove a non-empty directory should raise an exception - fileSystem.delete("/my_folder"); - } - - /** - * This tests uses ls - */ - @Test - public void testListDirectory() throws Exception { - fileSystem.createDirectory("/my_folder"); - fileSystem.createDirectory("/my_folder/my_sub_dir"); - fileSystem.write("/my_folder", "my_file", - new FileInputStream(randomFilePath)); - fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); - - List<String> lsResult = fileSystem.readDirectory("/my_folder"); - - // checking that result contains all the required data - assertTrue(lsResult.contains("my_sub_dir")); - assertTrue(lsResult.contains("my_file")); - assertTrue(lsResult.contains("my_link")); - - // ... and only those - assertEquals(3, lsResult.size()); - - lsResult = fileSystem.readDirectory("/"); - assertEquals(1, lsResult.size()); - } - - @Test - public void testConcurrency() throws Exception { - fileSystem.createDirectory("/mydir"); - try { - fileSystem.createDirectory("/myseconddir"); - } catch (ConcurrentModificationException e) { - fail(); - } - } - - @Test - public void testLinks() throws Exception { - fileSystem.createDirectory("/dir"); - fileSystem.createDirectory("/dir/subdir"); - fileSystem.createDirectory("/otherdir"); - - fileSystem.createSymbolicLink("/link", "dir/subdir"); - fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); - List<String> readResult = fileSystem.readDirectory("/dir/subdir"); - assertEquals(1, readResult.size()); - assertTrue(readResult.contains("subsubdirlink")); - - fileSystem.createDirectory("/link/subsubdirlink/finaldir"); - readResult = fileSystem.readDirectory("/otherdir"); - assertEquals(1, readResult.size()); - assertTrue(readResult.contains("finaldir")); - - } -} Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,89 +0,0 @@ -package org.nuiton.disworkfs; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.Socket; -import java.net.UnknownHostException; - -/** - * This class provides utilities to write tests easily. - * - * You can get multiples diswork configs ready to use. All the instances uses - * a new port and the local IP. - * <pre> - * c = newDisworkConfig(); // create a config for a bootstrap node - * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that - * // will bootstrap by joining the - * // first node - * c3 = newDisworkConfig(c.getUsedPort()) - * </pre> - * - */ -public class Util { - - protected static Integer port = 19000; - - /** - * returns a new port, returned value change at each call. - * @return - */ - public static Integer getPort() { - port += 1; - return port; - } - - /** - * returns the IP on the local machine. Trying to get an public IP or a LAN - * IP or the loopback IP if there is no other interface - * @return - * @throws UnknownHostException - */ - public static InetAddress getIp() throws UnknownHostException { - InetAddress result = InetAddress.getLocalHost(); - if (result.isLoopbackAddress()) { - try { - Socket temp = new Socket("microsoft.com", 80); - result = temp.getLocalAddress(); - temp.close(); - } catch (IOException e) { - // TODO 20100602 bleny do something ? - } - } - return result; - } - - /** - * returns a @link {@link DisworkFileSystemConfig} ready to be use as a - * config for a single-node instance of DisworkFS - * @return - * @throws UnknownHostException - */ - public static DisworkFileSystemConfig newDisworkConfig() - throws UnknownHostException { - return newDisworkConfig(null); - } - - /** - * returns a @link {@link DisworkFileSystemConfig} ready to be use as a - * config for a multiple-node instance of DisworkFS on a same machine. - * @param bootstrapPort the port on the same machine where another node can - * be found to bootstrap - * @return a complete config - * @throws UnknownHostException - */ - public static DisworkFileSystemConfig newDisworkConfig(Integer bootstrapPort) - throws UnknownHostException { - DisworkFileSystemConfig result = new DisworkFileSystemConfig(); - String port = Util.getPort().toString(); - String ip = getIp().getHostAddress(); - result.setOption("diswork.fs.use_in_memory_map", "false"); - result.setOption("diswork.fs.use_port", port); - result.setOption("diswork.fs.bootstrap.ip", ip); - if (bootstrapPort == null) { - result.setOption("diswork.fs.bootstrap.port", port); - } else { - result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); - } - return result; - } -} \ No newline at end of file Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,106 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +public abstract class AbstractDisworkMapTest { + + private final Log log = LogFactory.getLog(AbstractDisworkMapTest.class); + + + protected DisworkMap map1; + protected DisworkMap map2; + + @Before + public void setUp() throws Exception { + + } + + @After + public void tearDown() throws Exception { + /* + if (map1 != null) + map1.close(); + if (map2 != null) + map2.close(); + */ + } + + /** + * this test show that the InMemoryMap put implies a copy of the data. + * After the put, the original value is modified. A final check shows + * there is no side effect + */ + @Test + public void testPut() { + byte[] expected = {0x1, 0x2}; + map1.put("key", expected); + expected[0] = 0xf; + byte[] actual = map1.get("key"); + assertFalse(Arrays.equals(expected, actual)); + } + + + @Test + public void test() throws Exception { + DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); + + map1 = new KademliaDisworkMap(config); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + map1.put("test", newBytes); + + byte[] getResult = map1.get("test"); + assertArrayEquals(newBytes, getResult); + } + + @Test + public void test2nodes() throws Exception { + DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); + map1 = new KademliaDisworkMap(config1); + + DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); + map2 = new KademliaDisworkMap(config2); + + byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + log.info("trying put test"); + map1.put("test", newBytes); + + log.info("trying contains test"); + assertTrue(map2.containsKey("test")); + log.info("trying contains unknownkey"); + assertFalse(map2.containsKey("unknownkey")); + + log.info("trying get test"); + byte[] getResult = map2.get("test"); + + assertArrayEquals(newBytes, getResult); + + log.info("trying remove test"); + byte[] removeResult = map2.remove("test"); + assertArrayEquals(newBytes, removeResult); + + log.info("trying put test"); + map1.put("test", new byte[0]); + + log.info("trying put test"); + map2.put("test", new byte[1]); + + log.info("trying remove unknow"); + map2.remove("unknow"); + + } + +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java (from rev 60, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,14 @@ +package org.nuiton.disworkfs.storage; + +import org.junit.Before; + + +public class InMemoryDisworkMapTest extends AbstractDisworkMapTest { + + @Before + public void setUp() throws Exception { + map1 = new InMemoryDisworkMap(); + map2 = map1; + } + +} Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,27 +0,0 @@ -package org.nuiton.disworkfs.storage; - -import static org.junit.Assert.assertFalse; - -import java.util.Arrays; - -import org.junit.Test; -import org.nuiton.disworkfs.storage.InMemoryDisworkMap; - -public class InMemoryMapTest { - - /** - * this test show that the InMemoryMap put implies a copy of the data. - * After the put, the original value is modified. A final check shows - * there is no side effect - */ - @Test - public void testPut() { - InMemoryDisworkMap map = new InMemoryDisworkMap(); - byte[] expected = {0x1, 0x2}; - map.put("key", expected); - expected[0] = 0xf; - byte[] actual = map.get("key"); - assertFalse(Arrays.equals(expected, actual)); - } - -} Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -0,0 +1,31 @@ +package org.nuiton.disworkfs.storage; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; +import org.nuiton.disworkfs.DisworkFileSystemConfig; + +public class KademliaDisworkMapTest extends AbstractDisworkMapTest { + + @Before + public void setUp() throws Exception { + DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); + map1 = new KademliaDisworkMap(config1); + + DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newKademliaDisworkConfig(config1.getUsedPort()); + map2 = new KademliaDisworkMap(config2); + } + + @Test + public void testStringToIndentifier() { + assertNotNull(KademliaDisworkMap.stringToIdentifier("/")); + assertNotNull(KademliaDisworkMap.stringToIdentifier( + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" + /* + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir" + + "/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir/dir"*/)); + } +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-04 18:15:37 UTC (rev 62) @@ -1,98 +1,17 @@ package org.nuiton.disworkfs.storage; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import org.junit.After; import org.junit.Before; -import org.junit.Test; import org.nuiton.disworkfs.DisworkFileSystemConfig; -import org.nuiton.disworkfs.Util; -public class PastryDisworkMapTest { - - protected static PastryDisworkMap map1; - protected static PastryDisworkMap map2; +public class PastryDisworkMapTest extends AbstractDisworkMapTest { - protected static byte[] bytes; - @Before public void setUp() throws Exception { - } - - @After - public void tearDown() throws Exception { - /* - if (map1 != null) - map1.close(); - if (map2 != null) - map2.close(); - */ - } - - @Test - public void test() throws Exception { - DisworkFileSystemConfig config = Util.newDisworkConfig(); - - map1 = new PastryDisworkMap(config); - - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - bytes = newBytes; - - map1.put("test", newBytes); - - byte[] getResult = map1.get("test"); - assertArrayEquals(newBytes, getResult); - } - - @Test - public void test2nodes() throws Exception { - DisworkFileSystemConfig config1 = Util.newDisworkConfig(); + DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newPastryDisworkConfig(); map1 = new PastryDisworkMap(config1); - DisworkFileSystemConfig config2 = Util.newDisworkConfig(config1.getUsedPort()); + DisworkFileSystemConfig config2 = DisworkFileSystemConfig.newPastryDisworkConfig(config1.getUsedPort()); map2 = new PastryDisworkMap(config2); - - byte[] newBytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - bytes = newBytes; - - map1.put("test", newBytes); - - assertTrue(map2.containsKey("test")); - assertFalse(map2.containsKey("unknownkey")); - - byte[] getResult = map2.get("test"); - - assertArrayEquals(newBytes, getResult); - - byte[] removeResult = map2.remove("test"); - assertArrayEquals(newBytes, removeResult); - - map1.put("test", newBytes); - map2.put("test", newBytes); - - map2.remove("unknow"); - } - @Test - public void testClose() throws Exception { - /* - DisworkFileSystemConfig config1 = new DisworkFileSystemConfig(); - map1 = new PastryDisworkMap(config1); - */ - /* - try { - map2 = new PastryDisworkMap(config1); - } catch (IllegalStateException e) { - assertEquals("Cannot bind to /192.168.99.119:9001", e.getMessage()); - } - */ - - // map1.close(); - // map2 = new PastryDisworkMap(config1); - } - } Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-03 16:25:47 UTC (rev 61) +++ trunk/pom.xml 2010-06-04 18:15:37 UTC (rev 62) @@ -78,6 +78,11 @@ <artifactId>pastry</artifactId> <version>2.1</version> </dependency> + <dependency> + <groupId>org.planx.xmlstore</groupId> + <artifactId>xmlstore</artifactId> + <version>0.4.12</version> + </dependency>  <dependency>

1 0

r61 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 18:25:47 +0200 (Thu, 03 Jun 2010) New Revision: 61 Url: http://nuiton.org/repositories/revision/diswork/61 Log: bug fixes Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 16:25:47 UTC (rev 61) @@ -88,15 +88,14 @@ return finished; } - public void waitUntilFinised() { + public void waitUntilFinished() { while(!finished) { try { Thread.sleep(1500); log.info("waiting for response"); } catch (InterruptedException e) { - // TODO 20100527 bleny stub - log.info("exception catch", e); - e.printStackTrace(); + throw new RuntimeException + ("interrupted while waiting for response", e); } } } @@ -147,7 +146,12 @@ try { factory = new SocketPastryNodeFactory(nidFactory, bindport, env); } catch (BindException e) { - throw new IOException("unable to bind to port" + bindport); + // the bootstrap node can't be joined + throw new IOException(""/*"bootstrap node can't be joined"*/, e); + } catch (IllegalStateException e) { + // the bootstrap node can't be joined + throw new IOException + ("unable to bind to already used port " + bindport, e); } catch (ConnectException e) { // this occurs some times, it may work after some time... numberOfTry += 1; @@ -229,7 +233,7 @@ Id id = pastryIdFactory.buildId((String) key); ContainsKeyContinuation containsKeyContinuation = new ContainsKeyContinuation(); past.existsInOverlay(id, containsKeyContinuation); - containsKeyContinuation.waitUntilFinised(); + containsKeyContinuation.waitUntilFinished(); boolean result = containsKeyContinuation.result; log.info("containsKey " + key + " (id = " + id + ") returns " + result); return result; @@ -247,13 +251,19 @@ } finished = true; } + + @Override + public void receiveException(Exception exception) { + log.error("exception received while getting : " + exception); + finished = true; + } } protected byte[] atomicGet(Object key) { Id id = pastryIdFactory.buildId((String) key); GetContinuation getContinuation = new GetContinuation(); past.lookup(id, getContinuation); - getContinuation.waitUntilFinised(); + getContinuation.waitUntilFinished(); return getContinuation.result; } @@ -274,6 +284,12 @@ log.info("insert result received : " + Arrays.toString(result)); finished = true; } + + @Override + public void receiveException(Exception exception) { + log.error("exception received while inserting : " + exception); + finished = true; + } } protected void atomicPut(String key, byte[] value) { @@ -281,7 +297,7 @@ PastContent pastContent = new ByteArrayPastContent(id, value); PutContinuation putContinuation = new PutContinuation(); past.insert(pastContent, putContinuation); - putContinuation.waitUntilFinised(); + putContinuation.waitUntilFinished(); } @Override @@ -305,13 +321,19 @@ this.result = result; finished = true; } + + @Override + public void receiveException(Exception exception) { + log.error("exception received while removing : " + exception); + finished = true; + } } protected void atomicRemove(Object key) { Id id = pastryIdFactory.buildId((String) key); RemoveContinuation removeContinuation = new RemoveContinuation(); past.remove(id, removeContinuation); - removeContinuation.waitUntilFinised(); + removeContinuation.waitUntilFinished(); log.info("atomic remove " + key + " has returned " + removeContinuation.result); } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 15:09:54 UTC (rev 60) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 16:25:47 UTC (rev 61) @@ -308,6 +308,8 @@ map.put(key, map.get(newDataKey)); } + map.remove(newDataKey); + unLock(key); } else { Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-03 15:09:54 UTC (rev 60) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/Util.java 2010-06-03 16:25:47 UTC (rev 61) @@ -21,7 +21,7 @@ */ public class Util { - protected static Integer port = 9000; + protected static Integer port = 19000; /** * returns a new port, returned value change at each call.

1 0

r60 - in trunk: diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit diswork-fs/src/main/java/org/nuiton/disworkfs/storage src/license
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 17:09:54 +0200 (Thu, 03 Jun 2010) New Revision: 60 Url: http://nuiton.org/repositories/revision/diswork/60 Log: license headers, licence descriptor Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java trunk/src/license/project.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs; import java.io.IOException; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs; import java.io.Closeable; @@ -26,14 +50,12 @@ * </ul> */ public class DisworkFileSystem implements Closeable { + + private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - /** - * storage will permit to save and read directories, files and links - */ + /** storage will permit to save and read directories, files and links */ protected Storage storage; - private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - /** number of try to acquire a lock before giving up */ protected static final int LOCK_MAX_NUMBER_OF_TRY = 10; @@ -41,7 +63,7 @@ protected static final int LOCK_WAIT = 10 * 1000; // ten seconds /** default constructor (uses default configuration parameters) - * @throws IOException + * @throws IOException caused by network issue */ public DisworkFileSystem() throws IOException { this(new DisworkFileSystemConfig()); @@ -50,7 +72,7 @@ /** constructor allowing to provide another configuration to use * * @param config the configuration - * @throws IOException + * @throws IOException caused by network issue */ public DisworkFileSystem(DisworkFileSystemConfig config) throws IOException { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs; import org.nuiton.util.ApplicationConfig; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ /** * DisworkFS is a distributed file system. You can use it by instantiating * {@link org.nuiton.disworkfs.DisworkFileSystem} and use it to store Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.peerunit; import static fr.inria.peerunit.test.assertion.Assert.assertTrue; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.Closeable; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.IOException; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.IOException; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ package org.nuiton.disworkfs.storage; import java.io.ByteArrayInputStream; Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-03 15:09:54 UTC (rev 60) @@ -1,3 +1,27 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ /** * <p> * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a Modified: trunk/src/license/project.xml =================================================================== --- trunk/src/license/project.xml 2010-06-03 14:35:49 UTC (rev 59) +++ trunk/src/license/project.xml 2010-06-03 15:09:54 UTC (rev 60) @@ -9,7 +9,7 @@ <commentStyle>java</commentStyle> <fileSets> <fileSet> - <basedir>all/src</basedir> + <basedir>src/main/java</basedir> <includes> <include>**/*.java</include> </includes>

1 0

r59 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 16:35:49 +0200 (Thu, 03 Jun 2010) New Revision: 59 Url: http://nuiton.org/repositories/revision/diswork/59 Log: concurrency, - memory leak Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 13:11:32 UTC (rev 58) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 14:35:49 UTC (rev 59) @@ -137,9 +137,9 @@ if (EntryUtil.isDirectory(entryParent)) { String parentId = EntryUtil.getIdFromEntry(entryParent); + + // checking that file do not already exists in this directory String content = storage.getDirectory(parentId); - - // checking that file do not already exists in this directory String findResult = EntryUtil.findEntryInDirectory (content, fileName); if (findResult != null) { @@ -147,11 +147,8 @@ (parent + " already contains an element named " + fileName); } + // file do not exists, write file on the FS String newFileId = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.F, fileName, newFileId); - - // store file before meta info storage.putFile(newFileId, source); // update directory content @@ -160,6 +157,10 @@ while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { lockAcquired = storage.tryToLock(parentId); if (lockAcquired) { + // parent dir is locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); storage.putDirectory(parentId, newContent); storage.unLock(parentId); } else { @@ -167,7 +168,7 @@ try { Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { - log.info("exception catch", e); + log.info("wait for lock interrupted", e); throw new IOException ("interrupted while trying to acquire lock", e); } @@ -219,16 +220,49 @@ if (EntryUtil.isDirectory(entryParent)) { String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, dirName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + dirName); + } + + // store file before meta info String newDirectoryId = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.D, dirName, newDirectoryId); - - // store file before meta info storage.putDirectory(newDirectoryId, EntryUtil.EMPTY_DIRECTORY_CONTENT); - // update meta info directory - storage.putDirectory(parentId, newContent); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } } else if (EntryUtil.isLink(entryParent)) { String linkTarget = storage.getLink( EntryUtil.getIdFromEntry(entryParent)); @@ -282,19 +316,55 @@ } if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, name); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + name); + } + + // store file before meta info String newLinkId = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.L, name, newLinkId); + storage.putLink(newLinkId, target); - // store link before meta info - storage.putLink(newLinkId, target); - // update meta info directory - storage.putDirectory(parentId, newContent); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.L, name, newLinkId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { String linkTarget = storage.getLink( - EntryUtil.getIdFromEntry(entryParent)); + EntryUtil.getIdFromEntry(entryParent)); String newTarget = EntryUtil.resolveLink(parent, linkTarget); createSymbolicLink(newTarget, name, target); } else if (EntryUtil.isFile(entryParent)) { @@ -328,9 +398,6 @@ * @see #delete(String) */ protected void delete(String parent, String name) throws IOException { - // FIXME 20100602 bleny this implementation leaks - // only the association path => id of the metablock is removed, all - // block remains String entryParent = walk(parent); if (entryParent == null) { @@ -344,10 +411,9 @@ String entry = EntryUtil.findEntryInDirectory(content, name); String idToRemove = EntryUtil.getIdFromEntry(entry); - String newContent = EntryUtil.removeEntryFromEntries(content, name); - - // check if not removing a non-empty directory + // according to the type of the entry name, delete it if (EntryUtil.isDirectory(entry)) { + // check if not removing a non-empty directory String innerDirectoryId = EntryUtil.getIdFromEntry(entry); String innerDirectoryContent = storage.getDirectory(innerDirectoryId); @@ -358,13 +424,49 @@ throw new IOException ("trying to remove a non-empty directory"); } + + // remove it + storage.removeDirectory(idToRemove); + + } else if (EntryUtil.isFile(entry)) { + storage.removeFile(idToRemove); + } else if (EntryUtil.isLink(entry)) { + storage.removeLink(idToRemove); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } - // update meta info directory - storage.putDirectory(parentId, newContent); - // store file before meta info - storage.remove(idToRemove); + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.removeEntryFromEntries(content, name); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { String linkTarget = storage.getLink( EntryUtil.getIdFromEntry(entryParent)); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 13:11:32 UTC (rev 58) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 14:35:49 UTC (rev 59) @@ -9,8 +9,6 @@ import java.util.List; import java.util.Map; -import javax.swing.RowFilter.Entry; - import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -196,7 +194,22 @@ map.put(id, contentAsBytes); } + public void removeDirectory(String id) throws IOException { + log.debug("removeDirectory(\"" + id + "\")"); + removeKey(id); + } + public void removeFile(String id) throws IOException { + log.debug("removeFile(\"" + id + "\")"); + removeKey(id); + } + + public void removeLink(String id) { + log.debug("removeLink(\"" + id + "\")"); + map.remove(id); + } + + /** * see {@link #get(String)} */ @@ -213,12 +226,9 @@ * @param value * @throws IOException */ - protected void put(String key, InputStream value) + protected void put(String key, InputStream value) throws IOException, ConcurrentModificationException { - // TODO 20100602 bleny locks should be signed to prevent concurrency : - // due to latency, a lock claimed may be obtained even if someone tryed - // to obtain it before boolean lockAcquired = tryToLock(key); if (lockAcquired) { @@ -269,8 +279,11 @@ } eraseDependenciesOfMetablock(key); - map.put(key, map.get(newDataKey)); + if (isLockStillOwned(key)) { + map.put(key, map.get(newDataKey)); + } + unLock(key); } else { @@ -279,12 +292,18 @@ } + protected void remove(String key) throws IOException, + ConcurrentModificationException { + eraseDependenciesOfMetablock(key); + removeKey(key); + } + /** * due to concurrency, key should not be removed immediately. Someone may * read it now. It will be removed later, using {@link #clean()}. * @param key the key to be removed from the map */ - public void remove(String key) { + public void removeKey(String key) { idsToBeRemoved.add(key); } @@ -314,7 +333,7 @@ log.info("trying to acquire a lock on " + key); String lockKey = keyToLockKey(key); - byte[] lock = map.put(lockKey, EntryUtil.newLock(ownerId)); + byte[] lock = map.put(lockKey, EntryUtil.newLock(ownerId)); Boolean result = null; // if there was no lock or if current lock is mine @@ -356,7 +375,7 @@ * the value at this key has to be a metablock * @param key */ - public void eraseDependenciesOfMetablock(String key) { + protected void eraseDependenciesOfMetablock(String key) { byte[] value = map.get(key); if (value != null) { String obsoleteMetaBlock = EntryUtil.bytesToString(value); @@ -365,11 +384,19 @@ EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); log.info("removing " + obsoleteBlocksIds.length + " old blocks"); for (String obsoleteBlockId : obsoleteBlocksIds) { - map.remove(obsoleteBlockId); + removeKey(obsoleteBlockId); } } } } + + private boolean isLockStillOwned(String key) { + String lockKey = keyToLockKey(key); + byte[] lock = map.get(lockKey); + String lockOwner = EntryUtil.getOwnerFromLock(lock); + boolean isLockOwned = ownerId.equals(lockOwner); + return isLockOwned; + } protected boolean isObsolete(byte[] lock) { Long currentDate = System.currentTimeMillis(); Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 13:11:32 UTC (rev 58) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 14:35:49 UTC (rev 59) @@ -341,6 +341,6 @@ readResult = fileSystem.readDirectory("/otherdir"); assertEquals(1, readResult.size()); assertTrue(readResult.contains("finaldir")); - + } }

1 0

r58 - in trunk: diswork-fs/src/main/java/org/nuiton/disworkfs diswork-fs/src/main/java/org/nuiton/disworkfs/storage diswork-fs/src/test/java/org/nuiton/disworkfs diswork-fs/src/test/java/org/nuiton/disworkfs/storage src src/license
by bleny＠users.nuiton.org June 3, 2010

June 3, 2010

Author: bleny Date: 2010-06-03 15:11:32 +0200 (Thu, 03 Jun 2010) New Revision: 58 Url: http://nuiton.org/repositories/revision/diswork/58 Log: licence plugin config ; some bugfixs Added: trunk/src/license/ trunk/src/license/project.xml Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-06-03 13:11:32 UTC (rev 58) @@ -34,6 +34,12 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); + /** number of try to acquire a lock before giving up */ + protected static final int LOCK_MAX_NUMBER_OF_TRY = 10; + + /** time (ms) to wait between two try to acquire a lock */ + protected static final int LOCK_WAIT = 10 * 1000; // ten seconds + /** default constructor (uses default configuration parameters) * @throws IOException */ @@ -147,10 +153,42 @@ // store file before meta info storage.putFile(newFileId, source); - // update meta info directory - storage.putDirectory(parentId, newContent); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("exception catch", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + write(newTarget, fileName, source); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } } @@ -187,12 +225,20 @@ // store file before meta info storage.putDirectory(newDirectoryId, - EntryUtil.newEmptyDirectoryContent()); + EntryUtil.EMPTY_DIRECTORY_CONTENT); // update meta info directory storage.putDirectory(parentId, newContent); + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createDirectory(newTarget, dirName); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } } @@ -217,13 +263,16 @@ protected void createSymbolicLink(String parent, String name, String target) throws IOException { - if (target.startsWith("/")) { + if (target.startsWith(EntryUtil.PATH_SEPARATOR)) { // target is absolute } else { // target is relative, taking this in consideration target = EntryUtil.resolveLink(parent, target); } + log.info("trying to create symbolic link named " + name + " at path " + + parent + " with target \"" + target + "\""); + if (exists(target)) { String entryParent = walk(parent); @@ -243,8 +292,16 @@ storage.putLink(newLinkId, target); // update meta info directory storage.putDirectory(parentId, newContent); + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createSymbolicLink(newTarget, name, target); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } @@ -271,6 +328,9 @@ * @see #delete(String) */ protected void delete(String parent, String name) throws IOException { + // FIXME 20100602 bleny this implementation leaks + // only the association path => id of the metablock is removed, all + // block remains String entryParent = walk(parent); if (entryParent == null) { @@ -284,8 +344,7 @@ String entry = EntryUtil.findEntryInDirectory(content, name); String idToRemove = EntryUtil.getIdFromEntry(entry); - String newContent = EntryUtil.removeEntryFromEntries( - content, name); + String newContent = EntryUtil.removeEntryFromEntries(content, name); // check if not removing a non-empty directory if (EntryUtil.isDirectory(entry)) { @@ -294,7 +353,7 @@ storage.getDirectory(innerDirectoryId); // checking the emptiness of the directory if (!innerDirectoryContent.equals( - EntryUtil.newEmptyDirectoryContent())) { + EntryUtil.EMPTY_DIRECTORY_CONTENT)) { // directory is not empty throw new IOException ("trying to remove a non-empty directory"); @@ -306,8 +365,16 @@ // store file before meta info storage.remove(idToRemove); + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + delete(newTarget, name); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); } else { - throw new IOException(parent + " is not a directory"); + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); } } @@ -335,7 +402,7 @@ result = new ArrayList<String>(); String content = storage.getDirectory( EntryUtil.getIdFromEntry(entry)); - if (EntryUtil.newEmptyDirectoryContent().equals(content)) { + if (EntryUtil.EMPTY_DIRECTORY_CONTENT.equals(content)) { // directory is empty, add nothing } else { String[] entries = content.split( @@ -380,10 +447,10 @@ // FIXME 20105021 bleny works fine but is not understandable String result = null; - if (path.equals("/")) { + if (path.equals(EntryUtil.ROOT_DIRECTORY)) { return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "/" + EntryUtil.ENTRY_SEPARATOR - + "/"; + + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY; } String parentPath = EntryUtil.getParentFromPath(path); @@ -391,7 +458,7 @@ if (content == null) { // start of walk content = storage.getRootDirectory(); - result = walk(path, "/", content); + result = walk(path, EntryUtil.ROOT_DIRECTORY, content); } else if (parentPath.equals(current)) { // in the last directory // recuperation de l'element a traiter dans p @@ -414,7 +481,7 @@ String tail; // the path still remaining when in current - if (current.equals("/")) { + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { tail = path.substring(current.length()); } else { tail = path.substring(current.length() + 1); @@ -422,14 +489,16 @@ log.debug("current = " + current); log.debug("tail = " + tail); - String[] elementsNames = tail.split("/"); + String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); String p = elementsNames[0]; log.info("in intermediate dir " + current + ", looking for " + p); // mise a jour de current - if (current.equals("/")) current = ""; // avoid "//path" next line - current += "/" + p; + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + current = ""; // avoid "//path" next line + } + current += EntryUtil.PATH_SEPARATOR + p; String entry = EntryUtil.findEntryInDirectory(content, p); if (entry == null) { @@ -463,6 +532,6 @@ @Override public void close() throws IOException { storage.close(); - } + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-03 13:11:32 UTC (rev 58) @@ -32,21 +32,25 @@ public class EntryUtil { static public enum TYPE { /** Directory */ D, - /** Link */ L, - /** File */ F + /** Link */ L, + /** File */ F }; - static final int TYPE_LENGTH = 1; + static final public int TYPE_LENGTH = 1; static final public String ENTRY_SEPARATOR = ":"; static final public String ENTRIES_SEPARATOR = "\n"; static final public String BLOCKIDS_SEPARATOR = ";"; + static final public String PATH_SEPARATOR = "/"; + static final public String ROOT_DIRECTORY = "/"; + static final public String EMPTY_DIRECTORY_CONTENT = ""; + static final public String LOCK_SEPARATOR = "%"; - static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); + static final public int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); /** * This charset is forced in string-to-bytes and bytes-to-string conversions */ - static protected Charset CHARSET = Charset.forName("UTF-8"); + static final protected Charset CHARSET = Charset.forName("UTF-8"); /** * This is a utility class thus should not be instantiated @@ -141,13 +145,6 @@ /** * - */ - static public String newEmptyDirectoryContent() { - return ""; - } - - /** - * * @param content the string content of the directory * (all entries before add) * @param type the type of the new entry @@ -164,10 +161,15 @@ static public int getTotalSizeFromMetaBlock(String metaBlock) { // dealing with empty meta-block (for empty directory or file) - if (metaBlock.equals("0")) return 0; - String result = metaBlock.substring(0, - metaBlock.indexOf(BLOCKIDS_SEPARATOR)); - return Integer.valueOf(result); + Integer result = null; + if (metaBlock.equals("0")) { + result = 0; + } else { + String resultAsString = metaBlock.substring(0, + metaBlock.indexOf(BLOCKIDS_SEPARATOR)); + result = Integer.valueOf(resultAsString); + } + return result; } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { @@ -218,7 +220,7 @@ public static String getParentFromPath(String path) { String parent = FilenameUtils.getFullPathNoEndSeparator(path); if (parent.isEmpty()) { - parent = "/"; + parent = ROOT_DIRECTORY; } return parent; } @@ -227,4 +229,25 @@ return FilenameUtils.getName(path); } + public static String getOwnerFromLock(byte[] lock) { + String lockAsString = bytesToString(lock); + String lockInfo[] = lockAsString.split(LOCK_SEPARATOR); + String owner = lockInfo[0]; + return owner; + } + + public static Long getTimeFromLock(byte[] lock) { + String lockAsString = bytesToString(lock); + String lockInfo[] = lockAsString.split(LOCK_SEPARATOR); + Long lockAge = Long.parseLong(lockInfo[1]); + return lockAge; + } + + public static byte[] newLock(String ownerId) { + Long currentDate = System.currentTimeMillis(); + String lock = ownerId + LOCK_SEPARATOR + currentDate; + byte[] result = EntryUtil.stringToBytes(lock); + return result; + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-03 13:11:32 UTC (rev 58) @@ -1,6 +1,8 @@ package org.nuiton.disworkfs.storage; import java.io.IOException; +import java.net.BindException; +import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.Arrays; @@ -120,15 +122,17 @@ while(factory == null && numberOfTry <= 10) { try { factory = new SocketPastryNodeFactory(nidFactory, bindport, env); - } catch (java.net.ConnectException e) { + } catch (BindException e) { + throw new IOException("unable to bind to port" + bindport); + } catch (ConnectException e) { + // this occurs some times, it may work after some time... numberOfTry += 1; + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException ee) { + throw new IOException("Pastry boot process interrupted", ee); + } } - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - log.info("exception catch", e); - e.printStackTrace(); - } } if (factory == null) { throw new IOException("unable to connect"); @@ -147,16 +151,14 @@ try { node.wait(500); } catch (InterruptedException e) { - // TODO 20100528 bleny Auto-generated catch block - log.info("exception catch", e); - e.printStackTrace(); + throw new IOException("Pastry boot process interrupted", e); } // abort if can't join if (node.joinFailed()) { throw new IOException("Could not join the FreePastry ring. Reason:"+node.joinFailedReason()); } - } + } } log.info("finished creating new node " + node); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-03 13:11:32 UTC (rev 58) @@ -9,6 +9,8 @@ import java.util.List; import java.util.Map; +import javax.swing.RowFilter.Entry; + import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -27,6 +29,18 @@ protected static final long LOCK_VALID_TIME = 60 * 60 * 1000; + protected String ownerId = EntryUtil.generateId(); + + protected static String keyToLockKey(String key) { + return key + "_lock"; + } + + protected static String keyToNewDataKey(String key) { + return key + "_new_data"; + } + + // FIXME 20100602 bleny this collection should be stored somewhere in + // the map so another node will perform remove if this one fail protected List<String> idsToBeRemoved = new ArrayList<String>(); /** @@ -44,8 +58,7 @@ protected ByteArrayInputStream currentBlock; - protected int available; - + protected int numberOfAvailableBytes; public SplitBlocksInputStream(Map<String, byte[]> map, String id) { super(); @@ -75,11 +88,11 @@ } int result = currentBlock.read(); - available -= 1; + numberOfAvailableBytes -= 1; if(log.isDebugEnabled()) { log.debug("reading block number " + blockIdsIndex + - " (available = " + available + ")" + + " (available = " + numberOfAvailableBytes + ")" + " returns " + result); } return result; @@ -93,9 +106,9 @@ String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; - available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + numberOfAvailableBytes = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); log.debug("initializing stream with meta block \"" + metaBlock - + "\" (" + blockIds.length + " blocks, " + available + + "\" (" + blockIds.length + " blocks, " + numberOfAvailableBytes + " bytes)"); } } @@ -103,7 +116,7 @@ @Override public int available() { checkInitialization(); - return available; + return numberOfAvailableBytes; } } @@ -114,17 +127,21 @@ public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { this.disworkConfig = disworkConfig; + + // instantiating a map according to config directives if (disworkConfig.useInMemoryMap()) { log.info("using in-memory map"); - this.map = new InMemoryDisworkMap(); - putDirectory("/", EntryUtil.newEmptyDirectoryContent()); + map = new InMemoryDisworkMap(); } else { log.info("using Pastry map"); - this.map = new PastryDisworkMap(disworkConfig); - if (! map.containsKey("/")) { - putDirectory("/", EntryUtil.newEmptyDirectoryContent()); - } + map = new PastryDisworkMap(disworkConfig); } + + // creating root directory, if needed + if (! map.containsKey(EntryUtil.ROOT_DIRECTORY)) { + putDirectory(EntryUtil.ROOT_DIRECTORY, + EntryUtil.EMPTY_DIRECTORY_CONTENT); + } } /** @@ -132,7 +149,7 @@ * @throws IOException */ public String getRootDirectory() throws IOException { - String result = getDirectory("/"); + String result = getDirectory(EntryUtil.ROOT_DIRECTORY); return result; } @@ -189,7 +206,9 @@ } /** - * a put in the map, this involves split considerations and concurrency + * a put in the map, this involves split considerations and concurrency. + * a lock is acquired (it might have been acquired before). + * Lock is released at the end * @param key * @param value * @throws IOException @@ -197,90 +216,67 @@ protected void put(String key, InputStream value) throws IOException, ConcurrentModificationException { + // TODO 20100602 bleny locks should be signed to prevent concurrency : + // due to latency, a lock claimed may be obtained even if someone tryed + // to obtain it before + boolean lockAcquired = tryToLock(key); - String lockKey = key + "_lock"; - String newDataKey = key + "_newData"; - - // trying to acquire lock - Long currentDate = System.currentTimeMillis(); - log.info("trying to acquire a lock on " + key); - byte[] lock = map.put(lockKey, - EntryUtil.stringToBytes(currentDate.toString())); - - if (lock != null) { - // file is locked, check date - Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock)); - Long lockAge = currentDate - currentLock; - if (lockAge > LOCK_VALID_TIME) { - log.info("lock is out-dated (" + lockAge + " ms old)"); - // this lock is out-dated, let's erase all data - String obsoleteMetaBlock = - EntryUtil.bytesToString(map.get(newDataKey)); - if (obsoleteMetaBlock != null) { - String[] obsoleteBlocksIds = - EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); - log.info("removing " + obsoleteBlocksIds.length + " old blocks"); - for (String obsoleteBlockId : obsoleteBlocksIds) { - map.remove(obsoleteBlockId); - } - } - } else { - log.info(key + " is currently written, stopping operation"); - map.put(lockKey, lock); - throw new ConcurrentModificationException("can't write due to concurrency"); - } - } - - log.info("lock on " + key + " acquired"); - - // here, we know we can write or an exception - // would have been thrown earlier - String blocksIds = ""; - int readResult = 0; - int totalSize = 0; + if (lockAcquired) { + log.info("lock on " + key + " acquired"); - String metaBlock = totalSize + blocksIds; - map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); - - // creating a buffer of the size of a block - int bufferSize = disworkConfig.getBlockSize(); - byte[] buffer = new byte[bufferSize]; + // here, we know we can write or an exception + // would have been thrown earlier + String blocksIds = ""; + int readResult = 0; + int totalSize = 0; - while ((readResult = value.read(buffer)) != -1) { - totalSize += readResult; + String metaBlock = totalSize + blocksIds; + String newDataKey = keyToNewDataKey(key); + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // creating a buffer of the size of a block + int bufferSize = disworkConfig.getBlockSize(); + byte[] buffer = new byte[bufferSize]; - byte[] newBlock = buffer; + while ((readResult = value.read(buffer)) != -1) { + totalSize += readResult; - // if the buffer is not full, truncate the block - if (readResult < buffer.length) { - newBlock = new byte[readResult]; - System.arraycopy(buffer, 0, newBlock, 0, readResult); - } + byte[] newBlock = buffer; - String id = EntryUtil.generateId(); - blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + // if the buffer is not full, truncate the block + if (readResult < buffer.length) { + newBlock = new byte[readResult]; + System.arraycopy(buffer, 0, newBlock, 0, readResult); + } - log.debug("saving new block (size = " + newBlock.length + ")" + String id = EntryUtil.generateId(); + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + log.debug("saving new block (size = " + newBlock.length + ")" + " at key " + id); - // copy block in map - map.put(id, newBlock); + // copy block in map + map.put(id, newBlock); - metaBlock = totalSize + blocksIds; + metaBlock = totalSize + blocksIds; - log.debug("putting metablock " + metaBlock + " at key " + key); + log.debug("putting metablock " + metaBlock + " at key " + key); + + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // updating lock + updateLock(key); + } - map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + eraseDependenciesOfMetablock(key); + map.put(key, map.get(newDataKey)); - // updating lock - currentDate = System.currentTimeMillis(); - map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString())); + unLock(key); + + } else { + throw new ConcurrentModificationException("key " + key + " is locked"); } - map.put(key, map.get(newDataKey)); - - log.info("release lock " + lockKey); - map.remove(lockKey); } /** @@ -307,5 +303,79 @@ clean(); map.close(); } + + /** + * + * @param key + * @return true if lock acquired, false is lock is owned by another + */ + public boolean tryToLock(String key) { + // trying to acquire lock + log.info("trying to acquire a lock on " + key); + String lockKey = keyToLockKey(key); + byte[] lock = map.put(lockKey, EntryUtil.newLock(ownerId)); + + Boolean result = null; + // if there was no lock or if current lock is mine + if (lock == null || (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { + // file is not locked, we have took the lock + result = true; + } else { + // file is locked by other node, check date + if (isObsolete(lock)) { + log.info("lock is out-dated"); + // this lock is out-dated, let's erase all data + eraseNewData(key); + result = true; + } else { + log.info(key + " is currently written, stopping operation"); + map.put(lockKey, lock); + result = false; + } + } + return result; + } + + protected void updateLock(String key) { + String lockKey = keyToLockKey(key); + map.put(lockKey, EntryUtil.newLock(ownerId)); + } + + public void unLock(String key) { + String lockKey = keyToLockKey(key); + map.remove(lockKey); + } + + protected void eraseNewData(String key) { + String newDataKey = keyToNewDataKey(key); + eraseDependenciesOfMetablock(newDataKey); + } + + /** + * the value at this key has to be a metablock + * @param key + */ + public void eraseDependenciesOfMetablock(String key) { + byte[] value = map.get(key); + if (value != null) { + String obsoleteMetaBlock = EntryUtil.bytesToString(value); + if (obsoleteMetaBlock != null) { + String[] obsoleteBlocksIds = + EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); + log.info("removing " + obsoleteBlocksIds.length + " old blocks"); + for (String obsoleteBlockId : obsoleteBlocksIds) { + map.remove(obsoleteBlockId); + } + } + } + } + + protected boolean isObsolete(byte[] lock) { + Long currentDate = System.currentTimeMillis(); + Long currentLockTime = EntryUtil.getTimeFromLock(lock); + Long lockAge = currentDate - currentLockTime; + return lockAge > LOCK_VALID_TIME; + } + } \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java 2010-06-03 13:11:32 UTC (rev 58) @@ -17,15 +17,12 @@ public class DisworkFileSystemPastryTest extends DisworkFileSystemTest { + protected Integer bootstrapPort; + /** * this code executed after {@link DisworkFileSystemTest#setUp()} * @throws Exception */ - - private static final Log log = LogFactory.getLog(DisworkFileSystemPastryTest.class); - - protected Integer bootstrapPort; - @Before public void setUpFileSystem() throws Exception { // finally, initiate the fileSystem Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-06-03 13:11:32 UTC (rev 58) @@ -324,4 +324,23 @@ fail(); } } + + @Test + public void testLinks() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/otherdir"); + + fileSystem.createSymbolicLink("/link", "dir/subdir"); + fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); + List<String> readResult = fileSystem.readDirectory("/dir/subdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subsubdirlink")); + + fileSystem.createDirectory("/link/subsubdirlink/finaldir"); + readResult = fileSystem.readDirectory("/otherdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("finaldir")); + + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-03 10:23:17 UTC (rev 57) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-03 13:11:32 UTC (rev 58) @@ -130,6 +130,15 @@ } @Test + public void testGetParentFromPath() { + String path; + path = "/file"; + assertEquals("/", EntryUtil.getParentFromPath(path)); + path = "/dir/file"; + assertEquals("/dir", EntryUtil.getParentFromPath(path)); + } + + @Test public void testGetTotalSizeFromMetaBlock() { assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); } Added: trunk/src/license/project.xml =================================================================== --- trunk/src/license/project.xml (rev 0) +++ trunk/src/license/project.xml 2010-06-03 13:11:32 UTC (rev 58) @@ -0,0 +1,20 @@ +<?xml version='1.0' encoding='UTF-8'?> +<project xmlns="http://maven-site.nuiton.org/maven-license-plugin/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven-site.nuiton.org/maven-license-plugin/1.0.0 http://maven-site.nuiton.org/maven-license-plugin/licenseProjectDescriptor-…"> + <mainLicense>lgpl_v3</mainLicense> + <headers> + <header> + <licenseName>lgpl_v3</licenseName> + <commentStyle>java</commentStyle> + <fileSets> + <fileSet> + <basedir>all/src</basedir> + <includes> + <include>**/*.java</include> + </includes> + </fileSet> + </fileSets> + </header> + </headers> +</project>

1 0