Author: bleny Date: 2010-06-08 15:16:35 +0200 (Tue, 08 Jun 2010) New Revision: 68 Url: http://nuiton.org/repositories/revision/diswork/68 Log: changement du package de org.nuiton.disworkfs en org.nuiton.diswork.fs Added: trunk/diswork-fs/src/main/java/org/nuiton/diswork/ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/ trunk/diswork-fs/src/test/java/org/nuiton/diswork/ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/ Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/ Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/DisworkFileSystemTest.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/DisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/package-info.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/EntryUtilTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/PastryDisworkMapTest.java Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/Demo.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,214 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class Demo { + + private static final Log log = LogFactory.getLog(Demo.class); + + protected static final String USAGE = + "Usage :\n" + + "org.nuiton.disworkfs.Demo producer|consumer" + + " usePort [bootStrapIp bootStrapPort]"; + + protected static DisworkFileSystem fileSystem; + + protected static class Producer implements Runnable { + + protected String waitingForJob = null; + protected Integer expectedResult = null; + + @Override + public void run() { + while (true) { + try { + if (fileSystem.exists("/todo")) { + if (waitingForJob == null) { + Random random = new Random(); + waitingForJob = "/todo/job-" + random.nextInt(); + Integer randomInteger = random.nextInt(); + expectedResult = randomInteger + 1; + String job = randomInteger.toString(); + InputStream source = IOUtils.toInputStream(job); + log.info("writing job " + waitingForJob + " (" + + randomInteger + ")"); + fileSystem.write(waitingForJob, source); + source.close(); + } else { + String resultPath = waitingForJob + ".result"; + if (fileSystem.exists(resultPath)) { + log.info("results has been published"); + InputStream resultStream = + fileSystem.read(resultPath); + String resultString = + IOUtils.toString(resultStream); + Integer result = Integer.parseInt(resultString); + log.info("result is " + result + " (expected : " + + expectedResult + ")"); + fileSystem.delete(resultPath); + fileSystem.delete(waitingForJob); + waitingForJob = null; + expectedResult = null; + } + } + } else { + fileSystem.createDirectory("/todo"); + } + } catch (IOException e) { + log.error(e); + } + try { + Thread.sleep(15 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + } + } + } + + protected static class Consumer implements Runnable { + + @Override + public void run() { + while (true) { + try { + if (fileSystem.exists("/todo")) { + List<String> todos = fileSystem.readDirectory("/todo"); + if (todos.isEmpty()) { + log.info("nothing to do"); + } else { + // taking a random job + Random random = new Random(); + String todo = todos.get( + random.nextInt(todos.size()) + ); + if (!todo.endsWith(".result")) { + String jobPath = "/todo/" + todo; + log.info("reading the job " + jobPath); + InputStream in = fileSystem.read(jobPath); + String operation = IOUtils.toString(in); + // in.close(); + log.info("operation to do " + operation); + Integer i = Integer.parseInt(operation); + i += 1; + String result = i.toString(); + log.info("result is " + result); + InputStream source = + IOUtils.toInputStream(result); + fileSystem.write(jobPath + ".result", source); + source.close(); + } + } + } else { + fileSystem.createDirectory("/todo"); + } + } catch (IOException e) { + log.error(e); + } + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + log.info("sleep interrupted", e); + } + } + } + } + + public static void printUsage() { + System.out.println(USAGE); + } + + /** + * + * <dl> + * <dt>producer|consumer (required)</dt> + * <dd>a producer will produce operations todo, a consumer may read + * operations and try to execute them + * </dd> + * <dt>port (required)</dt> + * <dd>the port to use</dd> + * <dt>bootstrapip (optional)</dt> + * <dd>if not provided, the node will bootstrap itself, creating a + * new network. If provided, the node will first try to join a + * network</dd> + * <dt>bootstrap port (required if bootstrapip is set)</dt> + * <dd>the port used by the node to join</dd> + * </dl> + * + * Example : + * org.nuiton.disworkfs.Demo consumer 9001 + * org.nuiton.disworkfs.Demo producer 9002 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9003 127.0.0.1 9002 + * org.nuiton.disworkfs.Demo producer 9004 127.0.0.1 9001 + * org.nuiton.disworkfs.Demo consumer 9005 127.0.0.1 9003 + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + if (args.length == 2) { + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setOption("diswork.fs.use_port", args[1]); + fileSystem = new DisworkFileSystem(config); + if ("producer".equals(args[0])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[0])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + } else if (args.length == 4) { + DisworkFileSystemConfig config = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + config.setOption("diswork.fs.use_port", args[1]); + config.setOption("diswork.fs.bootstrap.ip", args[2]); + config.setOption("diswork.fs.bootstrap.port", args[3]); + fileSystem = new DisworkFileSystem(config); + if ("producer".equals(args[0])) { + Thread t = new Thread(new Producer()); + log.info("starting a producer"); + t.start(); + } else if ("consumer".equals(args[0])) { + Thread t = new Thread(new Consumer()); + log.info("starting a consumer"); + t.start(); + } + } else { + printUsage(); + } + } +} \ No newline at end of file Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,670 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.ConcurrentModificationException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.diswork.fs.storage.DisworkMap; +import org.nuiton.diswork.fs.storage.EntryUtil; +import org.nuiton.diswork.fs.storage.Storage; + +/** Main class of Diswork File System, provide methods for all operations + * You can use: + * <ul> + * <li>{@link #createDirectory(String)} and {@link #readDirectory(String)} + * to create and browse directories</li> + * <li>{@link #write(String, InputStream)} and {@link #read(String)} + * to write a file and read it</li> + * <li>{@link #createSymbolicLink(String, String)} to create symbolic links</li> + * <li>{@link #exists(String)} and {@link #delete(String)} can be used + * on directories, files and symlinks</li> + * </ul> + */ +public class DisworkFileSystem implements Closeable { + + private static final Log log = LogFactory.getLog(DisworkFileSystem.class); + + /** storage will permit to save and read directories, files and links */ + protected Storage storage; + + /** number of try to acquire a lock before giving up */ + protected static final int LOCK_MAX_NUMBER_OF_TRY = 10; + + /** time (ms) to wait between two try to acquire a lock */ + protected static final int LOCK_WAIT = 10 * 1000; // ten seconds + + /** default constructor (uses default configuration parameters) + * @throws IOException caused by network issue + */ + public DisworkFileSystem() throws IOException { + this(new DisworkFileSystemConfig()); + } + + /** constructor allowing to provide another configuration to use + * + * @param config the configuration + * @throws IOException caused by network issue + */ + public DisworkFileSystem(DisworkFileSystemConfig config) + throws IOException { + storage = new Storage(config); + } + + /** tests the existence of a file/dir/link at a given path + * return true if something exists at path p, it will be true if a call + * to <code>mkdir(p)</code>, <code>write(p, ?)</code> or + * <code>ln(p, ?)</code> has been done before. + * @param path a path in the virtual FS + * @return true is something (a link, a file, or a directory) exists at path + * @throws IOException + */ + public boolean exists(String path) throws IOException { + String entry = walk(path); + boolean result = entry != null; + return result; + } + + /** + * use this method to read the content of a file. A call of read may be + * done after a call to {@link #write} + * @param path the path to the file to read + * @return an InputStream on the file + * @throws FileNotFoundException if no file exists at this path + * @throws IOException if path exists but is a directory + */ + public InputStream read(String path) throws FileNotFoundException, + IOException { + + String entry = walk(path); + if (entry == null) { + throw new FileNotFoundException(path); + } + + InputStream result = null; + + if (EntryUtil.isLink(entry)) { + log.info("reading link " + path); + String id = EntryUtil.getIdFromEntry(entry); + String link = storage.getLink(id); + String newTarget = EntryUtil.resolveLink(path, link); + result = read(newTarget); + } else if (EntryUtil.isDirectory(entry)) { + throw new IOException("target is not a file: " + path); + } else if (EntryUtil.isFile(entry)) { + log.info("reading file " + path); + String id = EntryUtil.getIdFromEntry(entry); + result = storage.getFile(id); + } + + log.info("read " + path + " returns " + result.available() + " bytes"); + + return result; + } + + /** + * write a file on the file system at a given place + * @param path + * @param source + * @throws IOException if file already exists + * @throws ConcurrentModificationException if file is already being written + */ + public void write(String path, InputStream source) + throws IOException, + ConcurrentModificationException { + if (source == null) { + throw new IOException("source is not readable (null stream)"); + } + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + log.info("writing " + source.available() + " bytes at " + path); + write(parent, name, source); + } + + protected void write(String parent, String fileName, InputStream source) + throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); + + // checking that file do not already exists in this directory + String content = storage.getDirectory(parentId); + String findResult = EntryUtil.findEntryInDirectory + (content, fileName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + fileName); + } + + // file do not exists, write file on the FS + String newFileId = EntryUtil.generateId(); + storage.putFile(newFileId, source); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // parent dir is locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + write(newTarget, fileName, source); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + } + + /** + * creates a new empty directory + * @param path the absolute path (ending by the name) of the directory + * @throws IOException if parent path is not correct + */ + public void createDirectory(String path) + throws IOException, + ConcurrentModificationException { + String parent = EntryUtil.getParentFromPath(path); + String dirName = EntryUtil.getNameFromPath(path); + createDirectory(parent, dirName); + } + + protected void createDirectory(String parent, String dirName) + throws IOException { + + log.info("trying to create directory " + dirName + " in " + parent); + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); + String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, dirName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + dirName); + } + + // store file before meta info + String newDirectoryId = EntryUtil.generateId(); + storage.putDirectory(newDirectoryId, + EntryUtil.EMPTY_DIRECTORY_CONTENT); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createDirectory(newTarget, dirName); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + } + + /** + * create a new symbolic link + * @param path the path (ending by the name) where the link will be created + * @param target the path where the link point to + * (may be relative or absolute) + * @throws IOException if parent path is not correct + */ + public void createSymbolicLink(String path, String target) + throws IOException, + ConcurrentModificationException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + createSymbolicLink(parent, name, target); + } + + /** + * @see #createSymbolicLink(String, String) + */ + protected void createSymbolicLink(String parent, String name, String target) + throws IOException { + + if (target.startsWith(EntryUtil.PATH_SEPARATOR)) { + // target is absolute + } else { + // target is relative, taking this in consideration + target = EntryUtil.resolveLink(parent, target); + } + + log.info("trying to create symbolic link named " + name + " at path " + + parent + " with target \"" + target + "\""); + + if (exists(target)) { + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + + String parentId = EntryUtil.getIdFromEntry(entryParent); + String content = storage.getDirectory(parentId); + + // first, check if nothing already exists with this name + String findResult = EntryUtil.findEntryInDirectory + (content, name); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + name); + } + + // store file before meta info + String newLinkId = EntryUtil.generateId(); + storage.putLink(newLinkId, target); + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.L, name, newLinkId); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + createSymbolicLink(newTarget, name, target); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + + + } else { + throw new IOException(target + " is not a valid target"); + } + } + + /** + * remove a file, directory, or link. Non-empty directories can't be + * removed + * @param path the complete path to the entity to remove + * @throws IOException if path is incorrect or directory not empty + */ + public void delete(String path) throws IOException, + ConcurrentModificationException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + log.info("trying to remove " + path); + delete(parent, name); + } + + /** + * @see #delete(String) + */ + protected void delete(String parent, String name) throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getIdFromEntry(entryParent); + String content = storage.getDirectory(parentId); + + String entry = EntryUtil.findEntryInDirectory(content, name); + String idToRemove = EntryUtil.getIdFromEntry(entry); + + // according to the type of the entry name, delete it + if (EntryUtil.isDirectory(entry)) { + // check if not removing a non-empty directory + String innerDirectoryId = EntryUtil.getIdFromEntry(entry); + String innerDirectoryContent = + storage.getDirectory(innerDirectoryId); + // checking the emptiness of the directory + if (!innerDirectoryContent.equals( + EntryUtil.EMPTY_DIRECTORY_CONTENT)) { + // directory is not empty + throw new IOException + ("trying to remove a non-empty directory"); + } + + // remove it + storage.removeDirectory(idToRemove); + + } else if (EntryUtil.isFile(entry)) { + storage.removeFile(idToRemove); + } else if (EntryUtil.isLink(entry)) { + storage.removeLink(idToRemove); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + + + // update directory content + int numberOfTry = 0; + boolean lockAcquired = false; + while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { + lockAcquired = storage.tryToLock(parentId); + if (lockAcquired) { + // we have locked, do the update + content = storage.getDirectory(parentId); + String newContent = EntryUtil.removeEntryFromEntries(content, name); + storage.putDirectory(parentId, newContent); + storage.unLock(parentId); + } else { + log.info(parent + " is locked and can't be written"); + try { + Thread.sleep(LOCK_WAIT); + } catch (InterruptedException e) { + log.info("wait for lock interrupted", e); + throw new IOException + ("interrupted while trying to acquire lock", e); + } + } + } + + if (!lockAcquired) { + // fail, parent dir have not been written + throw new ConcurrentModificationException + ("can't write " + parent + " directory is locked"); + } + + } else if (EntryUtil.isLink(entryParent)) { + String linkTarget = storage.getLink( + EntryUtil.getIdFromEntry(entryParent)); + String newTarget = EntryUtil.resolveLink(parent, linkTarget); + delete(newTarget, name); + } else if (EntryUtil.isFile(entryParent)) { + throw new IOException(parent + " is not a directory"); + } else { + log.warn("strange entry" + entryParent); + throw new IOException("strange entry" + entryParent); + } + } + + /** + * list the content of a directory + * @param path the complete path to the directory + * @return a list of the names of the elements in <code>path</code> + * @throws IOException if path doesn't point to a directory + */ + public List<String> readDirectory(String path) throws IOException { + String entry = walk(path); + List<String> result = null; + + if (entry == null) { + throw new IOException(path + " doesn't exists"); + } else { + // path may be a link, if it's the case, + // entry become the actual directory + if (EntryUtil.isLink(entry)) { + String target = storage.getLink( + EntryUtil.getIdFromEntry(entry)); + return readDirectory(target); + } else if (EntryUtil.isDirectory(entry)) { + result = new ArrayList<String>(); + String content = storage.getDirectory( + EntryUtil.getIdFromEntry(entry)); + if (EntryUtil.EMPTY_DIRECTORY_CONTENT.equals(content)) { + // directory is empty, add nothing + } else { + String[] entries = content.split( + EntryUtil.ENTRIES_SEPARATOR); + for (String elementEntry : entries) { + result.add(EntryUtil.getNameFromEntry(elementEntry)); + } + } + } else if (EntryUtil.isFile(entry)) { + throw new IOException(path + " is not a directory but a file"); + } else { + log.warn("strange entry" + entry); + throw new IOException("strange entry" + entry); + } + } + + log.info("readDirectory " + path + " returns " + result.size() + " results"); + + return result; + } + + /** + * return the entry of the element at the end of <code>path</code> + * @param path + * @return null if path is not valid + */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + /** + * This method is a recursive function to walk through the tree structure + * of the directories and starting for root directory, following the + * symbolic links to reach the given path + * @param path + * @param current + * @param content + * @return null if path is not valid or the entry corresponding to path + * @throws IOException + */ + protected String walk(String path, String current, String content) + throws IOException { + // FIXME 20105021 bleny works fine but is not understandable + String result = null; + + // if path is "/", recursion can be initiated with this value : + // (it returns "/" as the id where to get the content of "/" + if (path.equals(EntryUtil.ROOT_DIRECTORY)) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY; + } + + String parentPath = EntryUtil.getParentFromPath(path); + + if (content == null) { + // start the recursion from root directory + content = storage.getRootDirectory(); + result = walk(path, EntryUtil.ROOT_DIRECTORY, content); + } else if (parentPath.equals(current)) { + // we are now in the last directory + // ie if path is a/b/p we are at a/b + + // p is the name of the element in a/b we search for + String tail = path.substring(current.length()); + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + + String entry = EntryUtil.findEntryInDirectory(content, p); + result = entry; + } else { + // in middle of path + // if path is a/b/c/d/e/f, we may be at b, c, d... + + String tail; // the path still remaining when in current + // ie if we are in c, tail is "d/e/f" + + // if we are at root directory, deal with the "/" + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + tail = path.substring(current.length()); + } else { + tail = path.substring(current.length() + 1); + } + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); + String p = elementsNames[0]; + + log.info("in intermediate dir " + current + ", looking for " + p); + + // updating current for recursion + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + current = ""; // avoid "//path" next line + } + current += EntryUtil.PATH_SEPARATOR + p; + + String entry = EntryUtil.findEntryInDirectory(content, p); + if (entry == null) { + result = null; + } else { + // we have found the entry to call recursion + + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + String linkContent = storage.getLink(id); + String newTarget = + EntryUtil.resolveLink(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else if (EntryUtil.isFile(entry)) { + // error, found file in path like '/dir1/dir2/filename/dir3' + result = null; + } else { + log.warn("strange case: " + entry); + result = null; + } + } + } + return result; + } + + @Override + public void close() throws IOException { + storage.close(); + } + + protected void setMap(DisworkMap map) { + storage.setMap(map); + } + +} Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,190 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.util.ApplicationConfig; + +/** + * This object contains parameters used in {@link DisworkFileSystem}. + * + * Available parameters are: + * <dl> + * <dt>blocks_size</dt> + * <dd> + * The size of blocks to create when splitting big data in bytes + * (by default, 10 MiB) + * </dd> + * </dl> + * + * + * This class provides utilities to write tests easily. + * + * You can get multiples diswork configs ready to use. All the instances uses + * a new port and the local IP. + * <pre> + * c = newDisworkConfig(); // create a config for a bootstrap node + * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that + * // will bootstrap by joining the + * // first node + * c3 = newDisworkConfig(c.getUsedPort()) + * </pre> + * + */ +public class DisworkFileSystemConfig extends ApplicationConfig { + + private static final Log log = + LogFactory.getLog(DisworkFileSystemConfig.class); + + protected static Integer port = 19000; + + /** + * returns a new port, returned value change at each call. + * @return + */ + public static Integer getPort() { + port += 1; + return port; + } + + /** + * returns the IP on the local machine. Trying to get an public IP or a LAN + * IP or the loopback IP if there is no other interface + * @return + * @throws UnknownHostException + */ + public static String getIp() throws UnknownHostException { + InetAddress result = InetAddress.getLocalHost(); + if (result.isLoopbackAddress()) { + try { + Socket temp = new Socket("microsoft.com", 80); + result = temp.getLocalAddress(); + temp.close(); + } catch (IOException e) { + log.warn("can't get external IP address"); + } + } + return result.getHostAddress(); + } + + public DisworkFileSystemConfig() { + setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB + + setDefaultOption("diswork.fs.map_type", "inmemory"); + setDefaultOption("diswork.fs.use_port", port.toString()); + } + + public int getBlockSize() { + return getOptionAsInt("diswork.fs.blocks_size"); + } + + public Integer getUsedPort() { + return getOptionAsInt("diswork.fs.use_port"); + } + + public void setUsedPort(Integer port) { + setOption("diswork.fs.use_port", port.toString()); + } + + public String getBootstrapIp() { + return getOption("diswork.fs.bootstrap.ip"); + } + + public void setBootstrapIp(String ip) { + setOption("diswork.fs.bootstrap.ip", ip); + } + + public Integer getBootstrapPort() { + return getOptionAsInt("diswork.fs.bootstrap.port"); + } + + public void setBootstrapPort(Integer port) { + setOption("diswork.fs.bootstrap.port", port.toString()); + } + + public String getMapType() { + return getOption("diswork.fs.map_type"); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a single-node instance of DisworkFS + * @return + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig() + throws UnknownHostException { + return newPastryDisworkConfig(null); + } + + /** + * returns a @link {@link DisworkFileSystemConfig} ready to be use as a + * config for a multiple-node instance of DisworkFS on a same machine. + * @param bootstrapPort the port on the same machine where another node can + * be found to bootstrap + * @return a complete config + * @throws UnknownHostException + */ + public static DisworkFileSystemConfig newPastryDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "pastry"); + result.setOption("diswork.fs.use_port", port); + result.setOption("diswork.fs.bootstrap.ip", ip); + if (bootstrapPort == null) { + result.setOption("diswork.fs.bootstrap.port", port); + } else { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + } + return result; + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig() + throws UnknownHostException { + return newKademliaDisworkConfig(null); + } + + public static DisworkFileSystemConfig newKademliaDisworkConfig(Integer bootstrapPort) + throws UnknownHostException { + DisworkFileSystemConfig result = new DisworkFileSystemConfig(); + String port = getPort().toString(); + String ip = getIp(); + result.setOption("diswork.fs.map_type", "kademlia"); + result.setOption("diswork.fs.use_port", port); + if (bootstrapPort != null) { + result.setOption("diswork.fs.bootstrap.port", bootstrapPort.toString()); + result.setOption("diswork.fs.bootstrap.ip", ip); + } + return result; + } +} Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java (from rev 66, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/package-info.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,35 @@ +/* + * #%L + * disworkfs + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +/** + * DisworkFS is a distributed file system. You can use it by instantiating + * {@link org.nuiton.diswork.fs.DisworkFileSystem} and use it to store + * directories and their content. + * + * You can change the default DisworkFileSystem behavior by provide a + * {@link org.nuiton.diswork.fs.DisworkFileSystemConfig} instance at + * construction. If you don't provide one, a instance will be created + */ + +package org.nuiton.diswork.fs; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/DisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/DisworkFileSystemTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/DisworkFileSystemTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.peerunit; +package org.nuiton.diswork.fs.peerunit; import static fr.inria.peerunit.test.assertion.Assert.assertTrue; import static fr.inria.peerunit.test.assertion.Assert.assertEquals; @@ -36,8 +36,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystem; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; import fr.inria.peerunit.TestCaseImpl; import fr.inria.peerunit.parser.TestStep; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/peerunit/package-info.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/peerunit/package-info.java 2010-06-08 13:16:35 UTC (rev 68) @@ -26,4 +26,4 @@ * In this package can be found some Test classes for PeerUnit framework */ -package org.nuiton.disworkfs.peerunit; \ No newline at end of file +package org.nuiton.diswork.fs.peerunit; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/DisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/DisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/DisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.Closeable; import java.util.Map; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/EntryUtil.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.nio.charset.Charset; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryDisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.IOException; import java.util.Arrays; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/KademliaDisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.DataInput; import java.io.DataInputStream; @@ -37,7 +37,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.planx.xmlstore.routing.Identifier; import org.planx.xmlstore.routing.Kademlia; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/PastryDisworkMap.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.IOException; import java.net.BindException; @@ -36,7 +36,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; import rice.Continuation; import rice.environment.Environment; Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-08 13:16:35 UTC (rev 68) @@ -22,7 +22,7 @@ * <http://www.gnu.org/licenses/lgpl-3.0.html>. * #L% */ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import java.io.ByteArrayInputStream; import java.io.Closeable; @@ -36,7 +36,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; /** * This class is the middle layer between the File System operations and Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/package-info.java 2010-06-08 13:16:35 UTC (rev 68) @@ -26,7 +26,7 @@ * <p> * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a * way to persistently store data. This is done by the - * {@link org.nuiton.disworkfs.storage.Storage} class + * {@link org.nuiton.diswork.fs.storage.Storage} class * which permit to store different type of data. * </p> * <p> @@ -108,13 +108,13 @@ * <strong>L</strong>ink), the name of the element, and an ID to be used as * a key on the map to get the actual content. Those three informations are * separated by ":" which is - * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRY_SEPARATOR}. + * {@link org.nuiton.diswork.fs.storage.EntryUtil#ENTRY_SEPARATOR}. * </p> * * <p> * A directory way have multiple <em>entries</em>. Entries of a directory are * separated by "\n" which is - * {@link org.nuiton.disworkfs.storage.EntryUtil#ENTRIES_SEPARATOR}. + * {@link org.nuiton.diswork.fs.storage.EntryUtil#ENTRIES_SEPARATOR}. * </p> * <p> * The above description shows the main principle used to store a tree @@ -198,7 +198,7 @@ * A metablock is composed of the total size of the file followed by * an ordered lists of IDs of the different blocks composing the file. * Those informations are separated by ";" (see - * {@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). + * {@link org.nuiton.diswork.fs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). * </p> * <p> * In the above example: @@ -222,10 +222,10 @@ * When reading and writing in storage, split is done transparently. When * reading, a Stream is returned: it loads data blocks after blocks * when needed (see - * {@link org.nuiton.disworkfs.storage.Storage.SplitBlocksInputStream}). + * {@link org.nuiton.diswork.fs.storage.Storage.SplitBlocksInputStream}). * When writing, data are split in blocks of a maximum configurable size * (see {@link org.nuiton.disworkfs.DisworkFileSystemConfig}). * </p> */ -package org.nuiton.disworkfs.storage; \ No newline at end of file +package org.nuiton.diswork.fs.storage; \ No newline at end of file Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AbstractDisworkFileSystemTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,361 @@ +package org.nuiton.diswork.fs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ConcurrentModificationException; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.util.FileUtil; + + +public abstract class AbstractDisworkFileSystemTest { + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = + System.getProperty("java.io.tmpdir", ".") // temp directory + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 9999; + + static protected DisworkFileSystem fileSystem; + + /** + * This setUp creates in a temp directory a file of size + * {@link randomFileSize} (fulfilling it with random bytes) + * This file can be found at {@link #randomFilePath} + * At the end of the method {@link #fileSystem} is ready to be used + * @throws Exception + */ + @Before + public void setUp() throws Exception { + // create a temp directory for our test + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + // creating random data for the file + Random random = new Random(); + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + /** + * writing a file at root directory should not raise any exception + * @throws Exception + */ + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + /** + * First, write a file then test if exists return true. + * @throws Exception + */ + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * tests {@link org.nuiton.diswork.fs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte: contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + source.close(); + readResult.close(); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testFailAtWrite() throws Exception { + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + } + + /** + * this use case should raise an exception because writing to a file + * that already exists + */ + @Test(expected = IOException.class) + public void testFailAtDoubleWrite() throws Exception { + InputStream source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + source.close(); + + source = new FileInputStream(randomFilePath); + try { + fileSystem.write("/my_file", source); + } finally { + source.close(); + } + } + + /** + * This test uses mkdir to create dirs and sub-dirs + * @throws Exception + */ + @Test + public void testCreateDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + /** + * Create some folders with mkdir and write files in those directories + * @throws Exception + */ + @Test + public void testWriteInFolder() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", + new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + /** + * create a symbolic link to a file. This test show that we can read the + * file using the link + * @throws Exception + */ + @Test + public void testLinking() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + /** + * Trying to create a link to a wrong target, this sould raise an exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtLinking() throws Exception { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + } + + /** + * Trying to remove files and directories + * @throws Exception + */ + @Test + public void testRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.delete("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.delete("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + /** + * By trying to remove a non-empty directory, this test should raise an + * exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.delete("/my_folder"); + } + + /** + * This tests uses ls + */ + @Test + public void testListDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.createDirectory("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); + + List<String> lsResult = fileSystem.readDirectory("/my_folder"); + + // checking that result contains all the required data + assertTrue(lsResult.contains("my_sub_dir")); + assertTrue(lsResult.contains("my_file")); + assertTrue(lsResult.contains("my_link")); + + // ... and only those + assertEquals(3, lsResult.size()); + + lsResult = fileSystem.readDirectory("/"); + assertEquals(1, lsResult.size()); + } + + @Test + public void testConcurrency() throws Exception { + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + } + + @Test + public void testLinks() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir/subdir"); + fileSystem.createDirectory("/otherdir"); + + fileSystem.createSymbolicLink("/link", "dir/subdir"); + fileSystem.createSymbolicLink("/link/subsubdirlink", "/otherdir"); + List<String> readResult = fileSystem.readDirectory("/dir/subdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subsubdirlink")); + + fileSystem.createDirectory("/link/subsubdirlink/finaldir"); + readResult = fileSystem.readDirectory("/otherdir"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("finaldir")); + + // tests that delete remove the link itself and not the target + fileSystem.delete("/link/subsubdirlink"); + assertTrue(fileSystem.exists("/otherdir")); + fileSystem.delete("/link"); + assertTrue(fileSystem.exists("/dir/subdir")); + + // testing the use of multiple links at the end of a path + fileSystem.createSymbolicLink("/link1", "/dir"); + fileSystem.createSymbolicLink("/link2", "/link1"); + fileSystem.createSymbolicLink("/link3", "/link2"); + + readResult = fileSystem.readDirectory("/link3"); + assertEquals(1, readResult.size()); + assertTrue(readResult.contains("subdir")); + } +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemInMemoryTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,21 @@ +package org.nuiton.diswork.fs; + +import org.junit.Before; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkFileSystemInMemoryTest extends AbstractDisworkFileSystemTest { + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); + disworkConfig.setOption("diswork.fs.map_type", "inmemory"); + fileSystem = new DisworkFileSystem(disworkConfig); + } + +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemKademliaTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,79 @@ +package org.nuiton.diswork.fs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkFileSystemKademliaTest extends AbstractDisworkFileSystemTest { + + protected Integer bootstrapPort; + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = + DisworkFileSystemConfig.newKademliaDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + } + + + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + + +} Copied: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java (from rev 66, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemPastryTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemPastryTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -0,0 +1,77 @@ +package org.nuiton.diswork.fs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.util.ConcurrentModificationException; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkFileSystemPastryTest extends AbstractDisworkFileSystemTest { + + protected Integer bootstrapPort; + + /** + * this code executed after {@link AbstractDisworkFileSystemTest#setUp()} + * @throws Exception + */ + @Before + public void setUpFileSystem() throws Exception { + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig1 = DisworkFileSystemConfig.newPastryDisworkConfig(); + bootstrapPort = disworkConfig1.getUsedPort(); + fileSystem = new DisworkFileSystem(disworkConfig1); + + } + + + @Test + public void testMultipleNodes1() throws Exception { + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + assertTrue(fileSystem.exists("/")); + assertTrue(fileSystem2.exists("/")); + } + + @Test + public void testMultipleNodes2() throws Exception { + DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newPastryDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + + byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + + assertTrue(fileSystem.exists("/my_file")); + assertTrue(fileSystem2.exists("/my_file")); + + assertEquals(1, fileSystem2.readDirectory("/").size()); + + byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + + assertArrayEquals(bytes, getResult); + + } + + @Test + public void testMultipleNodes3() throws Exception { + + fileSystem.createDirectory("/mydir"); + try { + fileSystem.createDirectory("/myseconddir"); + } catch (ConcurrentModificationException e) { + fail(); + } + + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/AbstractDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,4 +1,4 @@ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; @@ -11,6 +11,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.nuiton.diswork.fs.storage.DisworkMap; public abstract class AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/EntryUtilTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/EntryUtilTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -15,7 +15,7 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. *##%*/ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertArrayEquals; @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; import org.junit.Test; -import org.nuiton.disworkfs.storage.EntryUtil; +import org.nuiton.diswork.fs.storage.EntryUtil; /** * Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/InMemoryDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/InMemoryDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,6 +1,7 @@ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import org.junit.Before; +import org.nuiton.diswork.fs.storage.InMemoryDisworkMap; public class InMemoryDisworkMapTest extends AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/KademliaDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,4 +1,4 @@ -package org.nuiton.disworkfs.storage; +package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -10,7 +10,8 @@ import org.junit.Before; import org.junit.Test; -import org.nuiton.disworkfs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.storage.KademliaDisworkMap; import org.planx.xmlstore.routing.Identifier; public class KademliaDisworkMapTest extends AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/PastryDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/storage/PastryDisworkMapTest.java 2010-06-08 12:13:22 UTC (rev 66) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/PastryDisworkMapTest.java 2010-06-08 13:16:35 UTC (rev 68) @@ -1,3 +1,4 @@ +package org.nuiton.diswork.fs.storage; //package org.nuiton.disworkfs.storage; // //import org.junit.Before;