Author: tchemit Date: 2011-12-07 20:51:32 +0100 (Wed, 07 Dec 2011) New Revision: 2250 Url: http://nuiton.org/repositories/revision/nuiton-utils/2250 Log: Anomalie #1845: Can not import empty file Evolution #1846: Improve Import class code Added: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java =================================================================== --- trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java 2011-12-01 15:22:54 UTC (rev 2249) +++ trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java 2011-12-07 19:51:32 UTC (rev 2250) @@ -48,9 +48,9 @@ /** * Import engine for a given import model. - * + * <p/> * It acts as an {@link Iterable}, you can use directly inside a foreach. - * + * <p/> * The method {@link #prepareAndValidate()} will be invoked before all and * only once. It mainly obtain header from the csv input, pass it to the model * and then validate the model. @@ -84,8 +84,7 @@ return new Import<E>(model, inputStream); } - public static <E> Import<E> newImport(ImportModel<E> model, - Reader reader) { + public static <E> Import<E> newImport(ImportModel<E> model, Reader reader) { return new Import<E>(model, reader); } @@ -94,16 +93,17 @@ prepareAndValidate(); - readFirstLine(); - return new Iterator<E>() { - boolean hasNext = true; + // read first line since first line is header + boolean hasNext = readRow(); + // get once for all columns to import + List<ImportableColumn<E, Object>> columns = getNonIgnoredHeaders(); + + // to stock the current line number int lineNumber; - E lastElement; - @Override public boolean hasNext() { return hasNext; @@ -121,57 +121,22 @@ E element = model.newEmptyInstance(); - for (ImportableColumn<E, Object> field : getNonIgnoredHeaders()) { + for (ImportableColumn<E, Object> field : columns) { // read value from csv cell - String value; - try { - value = reader.get(field.getHeaderName()); - } catch (Exception e) { - reader.close(); - throw new ImportRuntimeException( - _("csv.import.error.unableToReadField", - field.getHeaderName(), lineNumber), e); - } - + String value = readValue(field, lineNumber); + // contravariance ftw - Object parsedValue; - try { - parsedValue = field.parseValue(value); - } catch (Exception e) { - String message = _("csv.import.error.unableToParseValue", - value, field.getHeaderName(), lineNumber) - + "\n" + e.getMessage(); - throw new ImportRuntimeException(message, e); - } - + Object parsedValue = parserValue(field, lineNumber, value); + // set value to element - try { - field.setValue(element, parsedValue); - } catch (Exception e) { - String message = _("csv.import.error.unableToSetValue", - parsedValue, - element.toString(), - lineNumber, field.getHeaderName()); - if (log.isErrorEnabled()) { - log.error(message); - } - throw new ImportRuntimeException(message, e); - } - - lastElement = element; + setValue(field, lineNumber, element, parsedValue); } - try { - hasNext = reader.readRecord(); - } catch (IOException e) { - reader.close(); - throw new ImportRuntimeException( - _("csv.import.error.unableToReadLine", - lineNumber + 1), e); - } + // check if there is a next row to read + hasNext = readRow(); - return lastElement; + return element; } @Override @@ -201,8 +166,7 @@ String[] headers = getHeaders(); if (log.isTraceEnabled()) { - log.trace("headers of the CSV file are : " + - Arrays.toString(headers)); + log.trace("headers of the CSV file are : " + Arrays.toString(headers)); } // hook to do some stuff from the model @@ -219,6 +183,50 @@ checkAllMandatoryHeadersArePresent(headers); } + protected <T> String readValue(ImportableColumn<E, T> field, + int lineNumber) { + try { + String value = reader.get(field.getHeaderName()); + return value; + } catch (Exception e) { + reader.close(); + throw new ImportRuntimeException( + _("csv.import.error.unableToReadField", + field.getHeaderName(), lineNumber), e); + } + } + + protected <T> T parserValue(ImportableColumn<E, T> field, + int lineNumber, String value) { + try { + T parsedValue = field.parseValue(value); + return parsedValue; + } catch (Exception e) { + String message = _("csv.import.error.unableToParseValue", + value, field.getHeaderName(), lineNumber) + + "\n" + e.getMessage(); + throw new ImportRuntimeException(message, e); + } + } + + protected <T> void setValue(ImportableColumn<E, T> field, + int lineNumber, + E element, + T parsedValue) { + try { + field.setValue(element, parsedValue); + } catch (Exception e) { + String message = _("csv.import.error.unableToSetValue", + parsedValue, + element.toString(), + lineNumber, field.getHeaderName()); + if (log.isErrorEnabled()) { + log.error(message); + } + throw new ImportRuntimeException(message, e); + } + } + protected void checkHeaderNamesAreAllKnown(String[] headers) { List<String> csvHeaders = new ArrayList<String>(); Collections.addAll(csvHeaders, headers); @@ -284,7 +292,7 @@ if (alreadyUsed) { throw new ImportRuntimeException( "model contains multiple columnsForImport named '" + - headerName + "'"); + headerName + "'"); } } } @@ -345,13 +353,21 @@ this.reader.setTrimWhitespace(true); } - protected void readFirstLine() throws ImportRuntimeException { + /** + * Reads a next row from the reader and return {@code true} if line + * was sucessfull read. + * + * @return {@code true} if line was sucessfull read, says in fact there is + * something after this line. + * @throws ImportRuntimeException if could not read line + */ + protected boolean readRow() throws ImportRuntimeException { try { - boolean emptyFile = !reader.readRecord(); - - if (emptyFile) { - throw new ImportRuntimeException("CSV file has no line"); - } + boolean hasNext = reader.readRecord(); + return hasNext; +// if (emptyFile) { +// throw new ImportRuntimeException("CSV file has no line"); +// } } catch (IOException e) { reader.close(); throw new ImportRuntimeException(_("csv.import.error.unableToReadLine", 1), e); Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java =================================================================== --- trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java 2011-12-01 15:22:54 UTC (rev 2249) +++ trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java 2011-12-07 19:51:32 UTC (rev 2250) @@ -24,19 +24,14 @@ */ package org.nuiton.util.csv; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import static org.nuiton.i18n.I18n._; - /** * A extended {@link Import} to read csv lines into a single map. * <p/> @@ -48,9 +43,6 @@ */ public class ImportToMap extends Import<Map<String, Object>> { - /** Logger. */ - private static final Log log = LogFactory.getLog(ImportToMap.class); - public static ImportToMap newImport(ImportModel<Map<String, Object>> model, InputStream inputStream) { return new ImportToMap(model, inputStream); @@ -67,16 +59,20 @@ // obtain headers from csv input and validate the model prepareAndValidate(); - // read first line since first line is header - readFirstLine(); - return new Iterator<Map<String, Object>>() { - boolean hasNext = true; + // read first line since first line is header + boolean hasNext = readRow(); + // get once for all columns to import + List<ImportableColumn<Map<String, Object>, Object>> columns = + getNonIgnoredHeaders(); + + // to stock the current line number int lineNumber; - Map<String, Object> element = new HashMap<String, Object>(); + // the map where to object of a row + final Map<String, Object> element = new HashMap<String, Object>(); @Override public boolean hasNext() { @@ -97,51 +93,20 @@ element.clear(); for (ImportableColumn<Map<String, Object>, Object> field : - getNonIgnoredHeaders()) { + columns) { // read value from csv cell - String value; - try { - value = reader.get(field.getHeaderName()); - } catch (Exception e) { - reader.close(); - throw new ImportRuntimeException(_("csv.import.error.unableToReadField", - field.getHeaderName(), lineNumber), e); - } + String value = readValue(field, lineNumber); // contravariance ftw - Object parsedValue; - try { - parsedValue = field.parseValue(value); - } catch (Exception e) { - String message = _("csv.import.error.unableToParseValue", - value, field.getHeaderName(), lineNumber) - + "\n" + e.getMessage(); - throw new ImportRuntimeException(message, e); - } + Object parsedValue = parserValue(field, lineNumber, value); // set value to element - try { - field.setValue(element, parsedValue); - } catch (Exception e) { - String message = _("csv.import.error.unableToSetValue", - parsedValue, - element.toString(), - lineNumber, field.getHeaderName()); - if (log.isErrorEnabled()) { - log.error(message); - } - throw new ImportRuntimeException(message, e); - } + setValue(field, lineNumber, element, parsedValue); } - try { - hasNext = reader.readRecord(); - } catch (IOException e) { - reader.close(); - throw new ImportRuntimeException( - _("csv.import.error.unableToReadLine", lineNumber + 1), e); - } + // check if there is a next row to read + hasNext = readRow(); return element; } Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java =================================================================== --- trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java 2011-12-01 15:22:54 UTC (rev 2249) +++ trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java 2011-12-07 19:51:32 UTC (rev 2250) @@ -28,8 +28,8 @@ import java.util.LinkedList; /** - * TODO * + * * @author bleny <leny@codelutin.com> * @author tchemit <chemit@codelutin.com> * @since 2.4 Modified: trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java =================================================================== --- trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java 2011-12-01 15:22:54 UTC (rev 2249) +++ trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java 2011-12-07 19:51:32 UTC (rev 2250) @@ -1,6 +1,9 @@ /* * #%L * Nuiton Utils :: Nuiton Csv + * + * $Id$ + * $HeadURL$ * %% * Copyright (C) 2011 CodeLutin * %% @@ -24,11 +27,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.nuiton.util.DateUtil; import org.nuiton.util.csv.Common.BeanProperty; +import java.io.IOException; import java.io.InputStream; import java.text.DateFormat; import java.text.ParseException; @@ -42,25 +45,50 @@ /** * Created on 28/11/11 * - * @author fdesbois <florian.desbois@wiztivi.com> + * @author fdesbois <desbois@codelutin.com> + * @author tchemit <chemit@codelutin.com> + * @since 2.4 */ public class ImportTest { - private DateFormat dateFormat; + private DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - @Before - public void setUp() { - dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + /** + * Test with {@link SimpleImportModel} that directly implements necessary {@link ImportableColumn} + * with a empty file (says the file has only the header definition but no row + * + * @throws Exception for errors + */ + @Test + public void testSimpleImportWithEmptyFile() throws Exception { + ImportModel<Row> model = new SimpleImportModel(dateFormat); + List<Row> rows = importFile(model, "emptyImportTest.csv"); + Assert.assertEquals(0, rows.size()); } /** + * Test with {@link SimpleImportModel} that directly implements necessary {@link ImportableColumn} + * with a empty file (says the file has only the header definition but no row + * + * @throws Exception for errors + */ + @Test + public void testSimpleImportWithEmptyFile2() throws Exception { + ImportModel<Row> model = new SimpleImportModel(dateFormat); + List<Row> rows = importFile(model, "emptyImport2Test.csv"); + Assert.assertEquals(0, rows.size()); + } + + /** * Test with {@link SimpleImportModel} that directly implements necessary {@link ImportableColumn}. * * @throws Exception for errors */ @Test public void testSimpleImport() throws Exception { - execute(new SimpleImportModel(dateFormat)); + ImportModel<Row> model = new SimpleImportModel(dateFormat); + List<Row> rows = importFile(model, "importTest.csv"); + checkImportTestResult(rows); } /** @@ -70,40 +98,45 @@ */ @Test public void testColumnImport() throws Exception { - execute(new ColumnImportModel(dateFormat)); + ImportModel<Row> model = new ColumnImportModel(dateFormat); + List<Row> rows = importFile(model, "importTest.csv"); + checkImportTestResult(rows); } - private void execute(ImportModel<Row> model) { - - InputStream csvStream = getClass().getResourceAsStream("importTest.csv"); - Import<Row> rowImport = Import.newImport(model, csvStream); + protected List<Row> importFile(ImportModel<Row> model, String filePath) throws IOException { + InputStream csvStream = getClass().getResourceAsStream(filePath); try { - Iterator<Row> it = rowImport.iterator(); - List<Row> rows = newList(it); - Assert.assertEquals(3, rows.size()); - Row row1 = rows.get(0); - Assert.assertEquals(DateUtil.createDate(5, 12, 2011), row1.getDate()); - Assert.assertEquals(new Integer(18), row1.getNumber()); - Assert.assertEquals("1ère ligne", row1.getTitle()); - Row row2 = rows.get(1); - Assert.assertEquals(DateUtil.createDate(6, 12, 2011), row2.getDate()); - Assert.assertEquals(new Integer(19), row2.getNumber()); - Assert.assertEquals("2ème ligne", row2.getTitle()); - Row row3 = rows.get(2); - Assert.assertEquals(DateUtil.createDate(7, 12, 2011), row3.getDate()); - Assert.assertEquals(new Integer(21), row3.getNumber()); - Assert.assertEquals("3ème ligne", row3.getTitle()); + Import<Row> rowImport = Import.newImport(model, csvStream); + try { + Iterator<Row> it = rowImport.iterator(); + List<Row> result = new ArrayList<Row>(); + while (it.hasNext()) { + result.add(it.next()); + } + return result; + } finally { + rowImport.close(); + } } finally { - rowImport.close(); + csvStream.close(); } } - private static <T> List<T> newList(Iterator<T> iterator) { - List<T> result = new ArrayList<T>(); - while (iterator.hasNext()) { - result.add(iterator.next()); - } - return result; + private void checkImportTestResult(List<Row> rows) { + + Assert.assertEquals(3, rows.size()); + Row row1 = rows.get(0); + Assert.assertEquals(DateUtil.createDate(5, 12, 2011), row1.getDate()); + Assert.assertEquals(new Integer(18), row1.getNumber()); + Assert.assertEquals("1ère ligne", row1.getTitle()); + Row row2 = rows.get(1); + Assert.assertEquals(DateUtil.createDate(6, 12, 2011), row2.getDate()); + Assert.assertEquals(new Integer(19), row2.getNumber()); + Assert.assertEquals("2ème ligne", row2.getTitle()); + Row row3 = rows.get(2); + Assert.assertEquals(DateUtil.createDate(7, 12, 2011), row3.getDate()); + Assert.assertEquals(new Integer(21), row3.getNumber()); + Assert.assertEquals("3ème ligne", row3.getTitle()); } public static class Row { @@ -234,7 +267,9 @@ @Override public void pushCsvHeaderNames(List<String> headerNames) { - log.info("Headers are : " + headerNames); + if (log.isDebugEnabled()) { + log.debug("Headers are : " + headerNames); + } } @Override Added: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv =================================================================== --- trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv (rev 0) +++ trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv 2011-12-07 19:51:32 UTC (rev 2250) @@ -0,0 +1 @@ +DATE;NUMBER;TITLE Property changes on: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv =================================================================== --- trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv (rev 0) +++ trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv 2011-12-07 19:51:32 UTC (rev 2250) @@ -0,0 +1 @@ +DATE;NUMBER;TITLE \ No newline at end of file Property changes on: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native
participants (1)
-
tchemit＠users.nuiton.org