Author: bleny Date: 2010-05-06 15:42:54 +0200 (Thu, 06 May 2010) New Revision: 25 Url: http://nuiton.org/repositories/revision/diswork/25 Log: merge SplitBytes/SplitFile synchro des thread, quitter le systeme proprement Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/BytesChunk.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitedFile.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplittedBytes.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AllwaysReplyToLookUpTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitedFileTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplittedBytesTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkServicesManager.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/SimpleDownload.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/AbstractDisworkService.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/DownloadService.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/UploadService.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileChunk.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileDescription.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromChunks.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromLocalFile.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/jgroups/JGroupsTransport.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitFileFromLocalFileTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-06 13:42:54 UTC (rev 25) @@ -2,7 +2,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -15,34 +14,19 @@ public class DisworkFileSystem { - private DownloadService downloadService; - private UploadService uploadService; - private LookUpService lookUpService; - private DisworkConfig disworkConfig; + protected DownloadService downloadService; + protected UploadService uploadService; + protected LookUpService lookUpService; + protected DisworkConfig disworkConfig; + protected DisworkServicesManager disworkServicesManager; private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) { this.disworkConfig = disworkConfig; - - // System.getProperty("user.home") - /* - String localIp; - try { - localIp = InetAddress.getLocalHost().getHostAddress(); - log.info("local IP is " + localIp); - } catch (UnknownHostException e) { - log.error("can't get local IP"); - } - */ - // System.setProperty("jgroups.bind_addr", localIp); - // System.setProperty("jgroups.tcpping.initial_hosts", localIp); - // System.setProperty("jgroups.udp.mcast_addr", "224.0.0.150"); - // System.setProperty("java.net.preferIPv4Stack", "true"); - - DisworkServicesManager disworkServicesManager = new DisworkServicesManager(disworkConfig); + disworkServicesManager = new DisworkServicesManager(disworkConfig); uploadService = new UploadService(); disworkServicesManager.register(uploadService); downloadService = new DownloadService(); @@ -72,27 +56,21 @@ } - OutputStream os = null; - try { - os = new FileOutputStream(file); - } catch (FileNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - return os; + return null; } public void write(String path, File source) throws IOException { File target = new File(disworkConfig.getStoragePath(), path); - /* - if (file.exists()) { - // TODO - throw new Exception("fichier existe deja"); - } - */ + if (target.exists()) { + throw new IOException(target.getAbsolutePath() + " already exists"); + } + FileUtil.copy(source, target); } + + public void close() { + disworkServicesManager.stop(); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkServicesManager.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkServicesManager.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkServicesManager.java 2010-05-06 13:42:54 UTC (rev 25) @@ -56,4 +56,15 @@ service.receiveMessage(message); } } + + /** + * this stop all the diswork services and close the + * ressources used by the transport layer + */ + public void stop() { + for (AbstractDisworkService service : services) { + service.stop(); + } + transport.close(); + } } \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/SimpleDownload.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/SimpleDownload.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/SimpleDownload.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,7 +1,5 @@ package org.nuiton.disworkfs; -import java.io.File; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.services.DownloadService; @@ -53,7 +51,7 @@ } - public File startDownload() throws InterruptedException { + public void startDownload() throws InterruptedException { if (log.isDebugEnabled()) log.info("starting download for " + fileDescription.getFileName()); @@ -64,9 +62,6 @@ if (log.isDebugEnabled()) log.info("download " + fileDescription.getFileName() + " is complete"); } - - // FIXME may not return a good file - return new File(fileDescription.getFileName()); } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/AbstractDisworkService.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/AbstractDisworkService.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/AbstractDisworkService.java 2010-05-06 13:42:54 UTC (rev 25) @@ -61,6 +61,8 @@ @Override public void run() {} + + public void stop() {} } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/DownloadService.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/DownloadService.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/DownloadService.java 2010-05-06 13:42:54 UTC (rev 25) @@ -35,7 +35,7 @@ log.info("received file chunk " + fileTransferMessage.getFileDescrition().getFileName() + " chunk number " - + fileTransferMessage.getFileChunk().getBytesChunk().getChunkNumber() + + fileTransferMessage.getFileChunk().getChunkNumber() + ")"); } @@ -70,7 +70,6 @@ // maybe the download is complete if (downloadingFile.isComplete()) { - // write file to the FS log.info("file " + newFile.getAbsolutePath() + " written"); finishedDownloads.add(checkSum); @@ -115,10 +114,7 @@ Message message = transport.newMulticastMessage(); message.setContent(new FileRequestMessage(fileDescription)); message.send(); - /* - Message message = new Message(null, null, new FileRequestMessage(fileDescription)); - jChannel.send(message); - */ + if (log.isDebugEnabled()) log.info("sending file request for " + fileDescription.getFileName()); registerObserver(fileDescription, downloadObserver); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/UploadService.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/UploadService.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/services/UploadService.java 2010-05-06 13:42:54 UTC (rev 25) @@ -53,7 +53,7 @@ reply.setContent(fileTransferMessage); if (log.isDebugEnabled()) - log.info("sending chunk " + fileTransferMessage.getFileChunk().getBytesChunk().getChunkNumber()); + log.info("sending chunk " + fileTransferMessage.getFileChunk().getChunkNumber()); reply.send(); Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/BytesChunk.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/BytesChunk.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/BytesChunk.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,49 +0,0 @@ -package org.nuiton.disworkfs.split; - -import java.io.Serializable; - -public class BytesChunk implements Serializable { - - /** - * - */ - private static final long serialVersionUID = 2562740291339559768L; - - // FIXME dangerous : not final ! - public static int MAX_CHUNK_SIZE = 1024; - - private byte[] data = new byte[MAX_CHUNK_SIZE]; - - private int chunkNumber; - - private int chunkSize; - - public BytesChunk(byte[] data) { - this.setData(data); - } - - public byte[] getData() { - return this.data; - } - - public void setData(byte[] data) { - this.data = data; - } - - public int getChunkNumber() { - return this.chunkNumber; - } - - public void setChunkNumber(int chunkNumber) { - this.chunkNumber = chunkNumber; - } - - public int getChunkSize() { - return this.chunkSize; - } - - public void setChunkSize(int chunkSize) { - this.chunkSize = chunkSize; - } - -} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileChunk.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileChunk.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileChunk.java 2010-05-06 13:42:54 UTC (rev 25) @@ -4,18 +4,54 @@ public class FileChunk implements Serializable { - /** - * - */ - private static final long serialVersionUID = 7668123461277672243L; - private BytesChunk bytesChunk; + private static final long serialVersionUID = 9081850894548724600L; - public FileChunk(BytesChunk bytesChunk) { - this.bytesChunk = bytesChunk; + // FIXME dangerous : not final ! + public static int MAX_CHUNK_SIZE = 1024; + + private byte[] data = new byte[MAX_CHUNK_SIZE]; + + private int chunkNumber; + + private int chunkSize; + + public FileChunk(byte[] data) { + this.setData(data); } - public BytesChunk getBytesChunk() { - return bytesChunk; + public byte[] getData() { + return this.data; } - + + public void setData(byte[] data) { + this.data = data; + } + + public int getChunkNumber() { + return this.chunkNumber; + } + + public void setChunkNumber(int chunkNumber) { + this.chunkNumber = chunkNumber; + } + + public int getChunkSize() { + return this.chunkSize; + } + + public void setChunkSize(int chunkSize) { + this.chunkSize = chunkSize; + } + + public static int numberOfChunksNeededtoStore(long numberOfBytes) { + long numberOfChunks = numberOfBytes / MAX_CHUNK_SIZE; + + if (numberOfBytes > numberOfChunks * MAX_CHUNK_SIZE) { + numberOfChunks += 1; + } + + // FIXME unsafe cast + return (int) numberOfChunks; + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileDescription.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileDescription.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/FileDescription.java 2010-05-06 13:42:54 UTC (rev 25) @@ -34,7 +34,7 @@ public int getNumberOfChunks() { - return SplittedBytes.numberOfChunksNeededtoStore(this.totalSize); + return FileChunk.numberOfChunksNeededtoStore(this.totalSize); } public void setFileName(String fileName) { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromChunks.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromChunks.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromChunks.java 2010-05-06 13:42:54 UTC (rev 25) @@ -84,9 +84,9 @@ public void addChunk(FileChunk fileChunk) { try { RandomAccessFile randomAccessFile = new RandomAccessFile(destination, "rw"); - byte[] data = fileChunk.getBytesChunk().getData(); - int off = BytesChunk.MAX_CHUNK_SIZE * fileChunk.getBytesChunk().getChunkNumber(); - int len = fileChunk.getBytesChunk().getChunkSize(); + byte[] data = fileChunk.getData(); + int off = FileChunk.MAX_CHUNK_SIZE * fileChunk.getChunkNumber(); + int len = fileChunk.getChunkSize(); randomAccessFile.seek(off); randomAccessFile.write(data, 0, len); randomAccessFile.close(); @@ -94,7 +94,7 @@ // updating status BitSet chunkStatus = readChunkStatusFile(); - chunkStatus.clear(fileChunk.getBytesChunk().getChunkNumber()); + chunkStatus.clear(fileChunk.getChunkNumber()); writeChunkStatusFile(chunkStatus); Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromLocalFile.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromLocalFile.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitFileFromLocalFile.java 2010-05-06 13:42:54 UTC (rev 25) @@ -33,7 +33,7 @@ List<FileChunk> result = new ArrayList<FileChunk>(); // this array will contains some bytes read from the file - byte[] read = new byte[BytesChunk.MAX_CHUNK_SIZE]; + byte[] read = new byte[FileChunk.MAX_CHUNK_SIZE]; // chunks have to be numbered int chunkNumber = 0; @@ -45,16 +45,15 @@ // reading the file until the end while ((chunkSize = randomAccessFile.read(read)) != -1) { - // creating a FileChunk from the data read - BytesChunk bytesChunk = new BytesChunk(read); - bytesChunk.setChunkNumber(chunkNumber); - bytesChunk.setChunkSize(chunkSize); + // creating a FileChunk from the data read + FileChunk fileChunk = new FileChunk(read); + fileChunk.setChunkNumber(chunkNumber); + fileChunk.setChunkSize(chunkSize); - FileChunk fileChunk = new FileChunk(bytesChunk); result.add(fileChunk); // preparing data for next iteration - read = new byte[BytesChunk.MAX_CHUNK_SIZE]; + read = new byte[FileChunk.MAX_CHUNK_SIZE]; chunkNumber += 1; } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitedFile.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitedFile.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplitedFile.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,3 +0,0 @@ -package org.nuiton.disworkfs.split; - - Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplittedBytes.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplittedBytes.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/SplittedBytes.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,167 +0,0 @@ -package org.nuiton.disworkfs.split; - -import java.util.ArrayList; -import java.util.List; - - -/** - * This class permit a great array of bytes (a file ?) to be split in multiple - * chunks. This class offers methods to split and unsplit an array of bytes. - * - * There is two use for this class : - * <ul> - * <li>Get a collection of chunks, given a single big array of bytes</li> - * <li>Get a single big array of bytes, given all the chunks</li> - * </ul> - * - * In the first case, you should use the SplittedBytes() constructor. Then, use - * the setChunksFromBytes method. Finally, use getChunks() to get all the - * generated chunks. - * - * In the second case, you can re-construct, step by step, adding chunks by - * chunks. First, use SplittedBytes(int expectedSize) to construct by providing - * the size of the array. Then, add chunks by chunks using addChunk. You don't - * need to provide the chunks in a particular order. It's useful if you're - * obtaining your chunks in an unknown order. You just have to provide all the - * chunks (until isComplete is true). Finally, get the final full array with - * getBytesFromChunks(). - * - * If you are missing some chunks, you can know who of them is missing with - * missingChinks() - * - * @author bleny - */ - - -public class SplittedBytes { - - private List<BytesChunk> chunks; - private long totalSize; - - public static int numberOfChunksNeededtoStore(long numberOfBytes) { - long longNumberOfChunks = numberOfBytes / BytesChunk.MAX_CHUNK_SIZE; - - // FIXME - int numberOfChunks = (int) longNumberOfChunks; - - if (numberOfBytes > numberOfChunks * BytesChunk.MAX_CHUNK_SIZE) { - numberOfChunks += 1; - } - return numberOfChunks; - } - - /** - * Use this constructor if you're building a byte-array from chunks. You - * have to know what will be the exact size of final array : this is needed - * to make isComplete working The argument is a number of bytes, not a - * number of chunks ! - * - * @param expectedSize - * the size of the array of bytes you will get - */ - public SplittedBytes(long expectedSize) { - this.totalSize = expectedSize; - - // create a list with one place for each chunks - List<BytesChunk> emptyChunkList = new ArrayList<BytesChunk>( - numberOfChunksNeededtoStore(expectedSize)); - for (int chunkNumber = 0; chunkNumber < numberOfChunksNeededtoStore(expectedSize); ++chunkNumber) { - emptyChunkList.add(null); - } - this.chunks = emptyChunkList; - } - - public SplittedBytes() { - this.chunks = new ArrayList<BytesChunk>(); - } - - public long getTotalSize() { - return this.totalSize; - } - - public void setChunksFromBytes(byte[] data) { - int numberOfChunks = data.length / BytesChunk.MAX_CHUNK_SIZE; - if (data.length > numberOfChunks * BytesChunk.MAX_CHUNK_SIZE) { - numberOfChunks += 1; - } - - for (int currentChunk = 0; currentChunk < numberOfChunks; ++currentChunk) { - - int numberOfLastBytes = BytesChunk.MAX_CHUNK_SIZE; - - if (data.length - (currentChunk * BytesChunk.MAX_CHUNK_SIZE) < BytesChunk.MAX_CHUNK_SIZE) { - // last chunk - numberOfLastBytes = data.length - currentChunk - * BytesChunk.MAX_CHUNK_SIZE; - } - - // reading numberOfLastBytes bytes - - byte[] chunkData = new byte[BytesChunk.MAX_CHUNK_SIZE]; - - System.arraycopy(data, currentChunk * BytesChunk.MAX_CHUNK_SIZE, - chunkData, 0, numberOfLastBytes); - - BytesChunk bytesChunk = new BytesChunk(chunkData); - bytesChunk.setChunkNumber(currentChunk); - bytesChunk.setChunkSize(numberOfLastBytes); - this.chunks.add(bytesChunk); - totalSize += numberOfLastBytes; - - } - } - - public List<BytesChunk> getChunks() { - return this.chunks; - } - - public byte[] getBytesFromChunks() throws Exception { - - if (!this.isComplete()) { - // TODO use a specific exception ? - throw new Exception("Data are incomplete : chunks are missing"); - } - - byte[] data = new byte[(int) totalSize]; - int bytesReads = 0; - totalSize = 0; - for (BytesChunk fileChunk : this.getChunks()) { - System.arraycopy(fileChunk.getData(), 0, data, bytesReads, - fileChunk.getChunkSize()); - - bytesReads += fileChunk.getChunkSize(); - totalSize += bytesReads; - } - - return data; - } - - public void addChunk(BytesChunk chunk) { - this.getChunks().set(chunk.getChunkNumber(), chunk); - } - - public boolean isComplete() { - return totalSize == this.getActualSize(); - } - - public int getActualSize() { - int result = 0; - for (BytesChunk bytesChunk : chunks) { - if (bytesChunk != null) { - result += bytesChunk.getChunkSize(); - } - } - return result; - } - - public List<Integer> missingChunks() { - List<Integer> missingChunks = new ArrayList<Integer>(); - for (int chunkNumber = 0; chunkNumber < this.getChunks().size(); ++chunkNumber) { - if (this.getChunks().get(chunkNumber) == null) { - missingChunks.add(chunkNumber); - } - } - return missingChunks; - } - -} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/Transport.java 2010-05-06 13:42:54 UTC (rev 25) @@ -39,5 +39,11 @@ * @return an address you can use in Message.setDestination() */ public Address getMulticastAddress(); + + /** + * this call should release all the ressources used by the transport layer + * i.e. close sockets, connections etc. + */ + public void close(); } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/jgroups/JGroupsTransport.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/jgroups/JGroupsTransport.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/transport/jgroups/JGroupsTransport.java 2010-05-06 13:42:54 UTC (rev 25) @@ -74,6 +74,21 @@ String clusterName = disworkConfig.getJGroupsClusterName(); + /* + String localIp; + try { + localIp = InetAddress.getLocalHost().getHostAddress(); + log.info("local IP is " + localIp); + } catch (UnknownHostException e) { + log.error("can't get local IP"); + } + */ + // System.setProperty("jgroups.bind_addr", localIp); + // System.setProperty("jgroups.tcpping.initial_hosts", localIp); + // System.setProperty("jgroups.udp.mcast_addr", "224.0.0.150"); + // System.setProperty("java.net.preferIPv4Stack", "true"); + + log.info("connecting to JGroup " + clusterName); jChannel.connect(clusterName); @@ -136,5 +151,13 @@ public Address getLocalAddress() { return new JGroupsAddress(jChannel.getAddress()); } + + @Override + public void close() { + // leaving group + jChannel.disconnect(); + // closing the channel + jChannel.close(); + } } Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-06 13:42:54 UTC (rev 25) @@ -6,6 +6,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level log4j.logger.org.nuiton.disworkfs=DEBUG -log4j.logger.org.nuiton.disworkfs.transport=WARN -log4j.logger.org.nuiton.disworkfs.services.DownloadService=WARN -log4j.logger.org.nuiton.disworkfs.services.UploadService=WARN +#log4j.logger.org.nuiton.disworkfs.transport=WARN +#log4j.logger.org.nuiton.disworkfs.services.DownloadService=WARN +#log4j.logger.org.nuiton.disworkfs.services.UploadService=WARN Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AllwaysReplyToLookUpTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AllwaysReplyToLookUpTest.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/AllwaysReplyToLookUpTest.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,34 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.mockito.Mockito.mock; - -import org.jgroups.JChannel; -import org.junit.Test; -import org.mockito.Mock; - -public class AllwaysReplyToLookUpTest extends AbstractSplitFileTest { - - @Mock JChannel mockedJChannel = mock(JChannel.class); - - - @Test - public void testReceive() { - // FIXME can't mock Message because getObject() is final - /* - - AllwaysReplyToLookUp allwaysReplyToLookUp = new AllwaysReplyToLookUp(mockedJChannel); - - Message mockedMessage = mock(Message.class); - // when(mockedMessage.getObject()).thenReturn(new LookUpMessage("randomFilePath")); - - - allwaysReplyToLookUp.receive(mockedMessage); - - - // verify(mockedJChannel, times(4)).send(mockedMessage); - - // when(message.getObject()).thenReturn(new LookUpMessage("randomFilePath")); - - */ - } -} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,9 +1,15 @@ package org.nuiton.disworkfs; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.Random; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -31,8 +37,8 @@ static protected String storagePath1; static protected String storagePath2; - static protected DisworkFileSystem disworkFileSystem1; - static protected DisworkFileSystem disworkFileSystem2; + static protected DisworkConfig disworkConfig1; + static protected DisworkConfig disworkConfig2; @Before public void setUp() throws Exception { @@ -57,13 +63,11 @@ FileUtils.writeByteArrayToFile(randomFile, randomBytes); - DisworkConfig disworkConfig1 = new DisworkConfig(); + disworkConfig1 = new DisworkConfig(); disworkConfig1.setOption("storage", storagePath1); - disworkFileSystem1 = new DisworkFileSystem(disworkConfig1); - DisworkConfig disworkConfig2 = new DisworkConfig(); + disworkConfig2 = new DisworkConfig(); disworkConfig2.setOption("storage", storagePath2); - disworkFileSystem2 = new DisworkFileSystem(disworkConfig2); } @After @@ -71,21 +75,46 @@ // cleaning FileUtil.deleteRecursively(tempDirectoryPath); } -/* + @Test public void testWrite() throws Exception { + DisworkFileSystem disworkFileSystem1 = new DisworkFileSystem(disworkConfig1); disworkFileSystem1.write("monfichier", new File(randomFilePath)); - + disworkFileSystem1.close(); } -*/ + + /** + * this test run two DistributedFS. A file is written on the first + * when. We try to read from the other FS : since it doesn't own the file, + * it will try a lookup and download the file. + */ @Test public void testRead() throws Exception { + DisworkFileSystem disworkFileSystem1 = new DisworkFileSystem(disworkConfig1); + DisworkFileSystem disworkFileSystem2 = new DisworkFileSystem(disworkConfig2); + disworkFileSystem1.write("monfichier", new File(randomFilePath)); - disworkFileSystem2.read("monfichier"); + // TODO + File monfichierstorage1 = new File(tempDirectoryPath + "/storage1/monfichier"); + File monfichierstorage2 = new File(tempDirectoryPath + "/storage2/monfichier"); + + try { + boolean actualContentEquality = IOUtils.contentEquals( + new FileInputStream(monfichierstorage1), + new FileInputStream(monfichierstorage2) + ); + assertTrue("file and copy content should be the same", actualContentEquality); + } catch (IOException e) { + fail("one or both files are not readable"); + e.printStackTrace(); + } + + disworkFileSystem1.close(); + disworkFileSystem2.close(); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitFileFromLocalFileTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitFileFromLocalFileTest.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitFileFromLocalFileTest.java 2010-05-06 13:42:54 UTC (rev 25) @@ -23,17 +23,17 @@ FileDescription fileDescription = splitFileFromLocalFile.getFileDescription(); assertTrue(fileDescription.getTotalSize() == randomFileSize); assertTrue(fileDescription.getFileCheckSum() != 0); - assertEquals(SplittedBytes.numberOfChunksNeededtoStore(randomFileSize), fileDescription.getNumberOfChunks()); + assertEquals(FileChunk.numberOfChunksNeededtoStore(randomFileSize), fileDescription.getNumberOfChunks()); assertEquals(fileDescription.getFileName(), randomFilePath); List<FileChunk> allChunks = splitFileFromLocalFile.getAllChunks(); for (FileChunk fileChunk : allChunks) { - int chunkSize = fileChunk.getBytesChunk().getChunkSize(); + int chunkSize = fileChunk.getChunkSize(); assertTrue( - chunkSize == BytesChunk.MAX_CHUNK_SIZE - || chunkSize == randomFileSize % BytesChunk.MAX_CHUNK_SIZE); + chunkSize == FileChunk.MAX_CHUNK_SIZE + || chunkSize == randomFileSize % FileChunk.MAX_CHUNK_SIZE); } } catch (IOException e) { Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitedFileTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitedFileTest.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplitedFileTest.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,115 +0,0 @@ -/*package org.nuiton.disworkfs.split; - - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.disworkfs.split.SplitedFile; - -public class SplitedFileTest { - - static Random random = new Random(); - - /** - * a place to store files for the test - * it's a subdirectory of the OS temp dir - * e.g. under linux /tmp/disworkfs/tests/ - - static String tempDirectoryPath = System.getProperty("java.io.tmpdir", ".") + "/disworkfs/tests"; - - *//** - * We will create a file at this path for test purpose - *//* - static String randomFilePath = tempDirectoryPath + "/randomfile"; - - *//** - * The file will have this fixed size - *//* - static int randomFileSize = 1 * 1000 * 1000; // 1 MB - - @Before - public void setUp() throws Exception { - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - // creating random data for the file - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - } - - @Test - public void testReadFromLocalFileSytem() { - try { - SplitedFile splitedFile = new SplitedFile(randomFilePath); - splitedFile.readFromLocalFileSytem(); - } catch (IOException e) { - fail(); - e.printStackTrace(); - } - } - - @Test - public void simpleCopy() { - - String splitedFileCopyPath = randomFilePath + "_copy"; - try { - // let's take a file - SplitedFile splitedFile = new SplitedFile(randomFilePath); - splitedFile.readFromLocalFileSytem(); - - - // let's create a second file - SplitedFile splitedFileCopy = new SplitedFile(splitedFileCopyPath); - - // here is the simple copy from fist file to second file - splitedFileCopy.setFileChunks(splitedFile.getFileChunks()); - - // write the copy to the FS - splitedFileCopy.writeToLocalFileSystem(); - - // compare the original and the copy byte by byte - try { - boolean actualContentEquality = IOUtils.contentEquals( - new FileInputStream(new File(randomFilePath)), - new FileInputStream(new File(splitedFileCopyPath)) - ); - assertTrue("file and copy content should be the same", actualContentEquality); - } catch (IOException e) { - fail("one or both files are not readable"); - e.printStackTrace(); - } - - // delete the copy - new File(splitedFileCopyPath).delete(); - } catch (IOException e) { - fail(); - e.printStackTrace(); - } catch (Exception e) { - fail(); - e.printStackTrace(); - } - } - - @After - public void tearDown() throws Exception { - // cleaning - new File(randomFilePath).delete(); - new File(tempDirectoryPath).delete(); - } - -} -*/ \ No newline at end of file Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplittedBytesTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplittedBytesTest.java 2010-05-06 09:48:34 UTC (rev 24) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/SplittedBytesTest.java 2010-05-06 13:42:54 UTC (rev 25) @@ -1,117 +0,0 @@ -package org.nuiton.disworkfs.split; - - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import org.junit.Before; -import org.junit.Test; - -public class SplittedBytesTest { - static final byte[] rawData = { 1, 2, 3, 4, 5, 6, 7, 8 }; - - static SplittedBytes splittedBytes = new SplittedBytes(); - - @Before - public void setUp() throws Exception { - BytesChunk.MAX_CHUNK_SIZE = 3; - } - - - /** - * this test show that bytes to chunks and chunks - * to bytes conversions are reflexive - * @throws Exception - */ - @Test - public void testSplitUnSplit() throws Exception { - splittedBytes.setChunksFromBytes(rawData); - - assertEquals(8, splittedBytes.getTotalSize()); - assertEquals(3, SplittedBytes.numberOfChunksNeededtoStore(rawData.length)); - - byte[] dataCopy = splittedBytes.getBytesFromChunks(); - - assertTrue("data after splitting and unplitting are not similar", Arrays.equals(rawData, dataCopy)); - - } - - /** - * This test shows that bytes can be split and - * unsplit by another instance of SplittedBytes - * @throws Exception - */ - @Test - public void testSplittedBytes2() throws Exception { - - SplittedBytes splittedBytes = new SplittedBytes(); - splittedBytes.setChunksFromBytes(rawData); - assertEquals(8, splittedBytes.getTotalSize()); - assertEquals(3, SplittedBytes.numberOfChunksNeededtoStore(rawData.length)); - - - - SplittedBytes otherSplittedBytes = new SplittedBytes(rawData.length); - for (BytesChunk bytesChunk : splittedBytes.getChunks()) { - assertFalse(otherSplittedBytes.isComplete()); - - int beforeAddActualSize = otherSplittedBytes.getActualSize(); - - otherSplittedBytes.addChunk(bytesChunk); - - int afterAddActualSize = otherSplittedBytes.getActualSize(); - - assertTrue("actualSize should progress", beforeAddActualSize < afterAddActualSize); - - } - - assertTrue(splittedBytes.isComplete()); - - byte[] dataCopy = splittedBytes.getBytesFromChunks(); - assertTrue("data after splitting and unplitting are not similar", Arrays.equals(rawData, dataCopy)); - - } - - /** - * This test shows that bytes can be split and - * unsplit by another instance of SplittedBytes - * @throws Exception - */ - @Test - public void testShuffleAdds() throws Exception { - - SplittedBytes splittedBytes = new SplittedBytes(); - splittedBytes.setChunksFromBytes(rawData); - - SplittedBytes otherSplittedBytes = new SplittedBytes(rawData.length); - - List<BytesChunk> someChunks = new ArrayList<BytesChunk>(); - - - // TODO use Collections.copy(someChunks, splittedBytes.getChunks()); - for (BytesChunk bytesChunk : splittedBytes.getChunks()) { - someChunks.add(bytesChunk); - } - - Collections.shuffle(someChunks); - - for (BytesChunk bytesChunk : someChunks) { - assertFalse(otherSplittedBytes.isComplete()); - otherSplittedBytes.addChunk(bytesChunk); - } - assertTrue(splittedBytes.isComplete()); - - byte[] dataCopy = splittedBytes.getBytesFromChunks(); - assertTrue("data after splitting and unplitting are not similar", Arrays.equals(rawData, dataCopy)); - - } - - - -}