[Nuiton-utils-commits] r2250 - in trunk/nuiton-csv/src: main/java/org/nuiton/util/csv test/java/org/nuiton/util/csv test/resources/org/nuiton/util/csv

7 Dec 2011

Author: tchemit
Date: 2011-12-07 20:51:32 +0100 (Wed, 07 Dec 2011)
New Revision: 2250

Url: http://nuiton.org/repositories/revision/nuiton-utils/2250

Log:
Anomalie #1845: Can not import empty file
Evolution #1846: Improve Import class code


Added:
   trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv
   trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv
Modified:
   trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java
   trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java
   trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java
   trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java

Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java
===================================================================

--- trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java	2011-12-01 15:22:54 UTC (rev 2249)
+++ trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/Import.java	2011-12-07 19:51:32 UTC (rev 2250)
@@ -48,9 +48,9 @@
 
 /**
  * Import engine for a given import model.
- *
+ * <p/>
  * It acts as an {@link Iterable}, you can use directly inside a foreach.
- *
+ * <p/>
  * The method {@link #prepareAndValidate()} will be invoked before all and
  * only once. It mainly obtain header from the csv input, pass it to the model
  * and then validate the model.
@@ -84,8 +84,7 @@
         return new Import<E>(model, inputStream);
     }
 
-    public static <E> Import<E> newImport(ImportModel<E> model,
-                                          Reader reader) {
+    public static <E> Import<E> newImport(ImportModel<E> model, Reader reader) {
         return new Import<E>(model, reader);
     }
 
@@ -94,16 +93,17 @@
 
         prepareAndValidate();
 
-        readFirstLine();
-
         return new Iterator<E>() {
 
-            boolean hasNext = true;
+            // read first line since first line is header
+            boolean hasNext = readRow();
 
+            // get once for all columns to import
+            List<ImportableColumn<E, Object>> columns = getNonIgnoredHeaders();
+
+            // to stock the current line number
             int lineNumber;
 
-            E lastElement;
-
             @Override
             public boolean hasNext() {
                 return hasNext;
@@ -121,57 +121,22 @@
 
                 E element = model.newEmptyInstance();
 
-                for (ImportableColumn<E, Object> field : getNonIgnoredHeaders()) {
+                for (ImportableColumn<E, Object> field : columns) {
 
                     // read value from csv cell
-                    String value;
-                    try {
-                        value = reader.get(field.getHeaderName());
-                    } catch (Exception e) {
-                        reader.close();
-                        throw new ImportRuntimeException(
-                                _("csv.import.error.unableToReadField",
-                                  field.getHeaderName(), lineNumber), e);
-                    }
-
+                    String value = readValue(field, lineNumber);
+                    
                     // contravariance ftw
-                    Object parsedValue;
-                    try {
-                        parsedValue = field.parseValue(value);
-                    } catch (Exception e) {
-                        String message = _("csv.import.error.unableToParseValue",
-                                           value, field.getHeaderName(), lineNumber)
-                                         + "\n" + e.getMessage();
-                        throw new ImportRuntimeException(message, e);
-                    }
-
+                    Object parsedValue = parserValue(field, lineNumber, value);
+                    
                     // set value to element
-                    try {
-                        field.setValue(element, parsedValue);
-                    } catch (Exception e) {
-                        String message = _("csv.import.error.unableToSetValue",
-                                           parsedValue,
-                                           element.toString(),
-                                           lineNumber, field.getHeaderName());
-                        if (log.isErrorEnabled()) {
-                            log.error(message);
-                        }
-                        throw new ImportRuntimeException(message, e);
-                    }
-
-                    lastElement = element;
+                    setValue(field, lineNumber, element, parsedValue);
                 }
 
-                try {
-                    hasNext = reader.readRecord();
-                } catch (IOException e) {
-                    reader.close();
-                    throw new ImportRuntimeException(
-                            _("csv.import.error.unableToReadLine",
-                              lineNumber + 1), e);
-                }
+                // check if there is a next row to read
+                hasNext = readRow();
 
-                return lastElement;
+                return element;
             }
 
             @Override
@@ -201,8 +166,7 @@
         String[] headers = getHeaders();
 
         if (log.isTraceEnabled()) {
-            log.trace("headers of the CSV file are : " +
-                      Arrays.toString(headers));
+            log.trace("headers of the CSV file are : " + Arrays.toString(headers));
         }
 
         // hook to do some stuff from the model
@@ -219,6 +183,50 @@
         checkAllMandatoryHeadersArePresent(headers);
     }
 
+    protected <T> String readValue(ImportableColumn<E, T> field,
+                               int lineNumber) {
+        try {
+            String value = reader.get(field.getHeaderName());
+            return value;
+        } catch (Exception e) {
+            reader.close();
+            throw new ImportRuntimeException(
+                    _("csv.import.error.unableToReadField",
+                      field.getHeaderName(), lineNumber), e);
+        }
+    }
+
+    protected <T> T parserValue(ImportableColumn<E, T> field,
+                                 int lineNumber, String value) {
+        try {
+            T parsedValue = field.parseValue(value);
+            return  parsedValue;
+        } catch (Exception e) {
+            String message = _("csv.import.error.unableToParseValue",
+                               value, field.getHeaderName(), lineNumber)
+                    + "\n" + e.getMessage();
+            throw new ImportRuntimeException(message, e);
+        }
+    }
+
+    protected <T> void setValue(ImportableColumn<E, T> field,
+                                int lineNumber,
+                                E element,
+                                T parsedValue) {
+        try {
+            field.setValue(element, parsedValue);
+        } catch (Exception e) {
+            String message = _("csv.import.error.unableToSetValue",
+                               parsedValue,
+                               element.toString(),
+                               lineNumber, field.getHeaderName());
+            if (log.isErrorEnabled()) {
+                log.error(message);
+            }
+            throw new ImportRuntimeException(message, e);
+        }
+    }
+
     protected void checkHeaderNamesAreAllKnown(String[] headers) {
         List<String> csvHeaders = new ArrayList<String>();
         Collections.addAll(csvHeaders, headers);
@@ -284,7 +292,7 @@
             if (alreadyUsed) {
                 throw new ImportRuntimeException(
                         "model contains multiple columnsForImport named '" +
-                        headerName + "'");
+                                headerName + "'");
             }
         }
     }
@@ -345,13 +353,21 @@
         this.reader.setTrimWhitespace(true);
     }
 
-    protected void readFirstLine() throws ImportRuntimeException {
+    /**
+     * Reads a next row from the reader and return {@code true} if line
+     * was sucessfull read.
+     *
+     * @return {@code true} if line was sucessfull read, says in fact there is
+     * something after this line.
+     * @throws ImportRuntimeException if could not read line
+     */
+    protected boolean readRow() throws ImportRuntimeException {
         try {
-            boolean emptyFile = !reader.readRecord();
-
-            if (emptyFile) {
-                throw new ImportRuntimeException("CSV file has no line");
-            }
+            boolean hasNext = reader.readRecord();
+            return hasNext;
+//            if (emptyFile) {
+//                throw new ImportRuntimeException("CSV file has no line");
+//            }
         } catch (IOException e) {
             reader.close();
             throw new ImportRuntimeException(_("csv.import.error.unableToReadLine", 1), e);

Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java
===================================================================
--- trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java	2011-12-01 15:22:54 UTC (rev 2249)
+++ trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ImportToMap.java	2011-12-07 19:51:32 UTC (rev 2250)
@@ -24,19 +24,14 @@
  */
 package org.nuiton.util.csv;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.io.IOException;
 import java.io.InputStream;
 import java.io.Reader;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 
-import static org.nuiton.i18n.I18n._;
-
 /**
  * A extended {@link Import} to read csv lines into a single map.
  * <p/>
@@ -48,9 +43,6 @@
  */
 public class ImportToMap extends Import<Map<String, Object>> {
 
-    /** Logger. */
-    private static final Log log = LogFactory.getLog(ImportToMap.class);
-
     public static ImportToMap newImport(ImportModel<Map<String, Object>> model,
                                         InputStream inputStream) {
         return new ImportToMap(model, inputStream);
@@ -67,16 +59,20 @@
         // obtain headers from csv input and validate the model
         prepareAndValidate();
 
-        // read first line since first line is header
-        readFirstLine();
-
         return new Iterator<Map<String, Object>>() {
 
-            boolean hasNext = true;
+            // read first line since first line is header
+            boolean hasNext = readRow();
 
+            // get once for all columns to import
+            List<ImportableColumn<Map<String, Object>, Object>> columns =
+                    getNonIgnoredHeaders();
+
+            // to stock the current line number
             int lineNumber;
 
-            Map<String, Object> element = new HashMap<String, Object>();
+            // the map where to object of a row
+            final Map<String, Object> element = new HashMap<String, Object>();
 
             @Override
             public boolean hasNext() {
@@ -97,51 +93,20 @@
                 element.clear();
 
                 for (ImportableColumn<Map<String, Object>, Object> field :
-                        getNonIgnoredHeaders()) {
+                        columns) {
 
                     // read value from csv cell
-                    String value;
-                    try {
-                        value = reader.get(field.getHeaderName());
-                    } catch (Exception e) {
-                        reader.close();
-                        throw new ImportRuntimeException(_("csv.import.error.unableToReadField",
-                                                           field.getHeaderName(), lineNumber), e);
-                    }
+                    String value = readValue(field, lineNumber);
 
                     // contravariance ftw
-                    Object parsedValue;
-                    try {
-                        parsedValue = field.parseValue(value);
-                    } catch (Exception e) {
-                        String message = _("csv.import.error.unableToParseValue",
-                                           value, field.getHeaderName(), lineNumber)
-                                         + "\n" + e.getMessage();
-                        throw new ImportRuntimeException(message, e);
-                    }
+                    Object parsedValue = parserValue(field, lineNumber, value);
 
                     // set value to element
-                    try {
-                        field.setValue(element, parsedValue);
-                    } catch (Exception e) {
-                        String message = _("csv.import.error.unableToSetValue",
-                                           parsedValue,
-                                           element.toString(),
-                                           lineNumber, field.getHeaderName());
-                        if (log.isErrorEnabled()) {
-                            log.error(message);
-                        }
-                        throw new ImportRuntimeException(message, e);
-                    }
+                    setValue(field, lineNumber, element, parsedValue);
                 }
 
-                try {
-                    hasNext = reader.readRecord();
-                } catch (IOException e) {
-                    reader.close();
-                    throw new ImportRuntimeException(
-                            _("csv.import.error.unableToReadLine", lineNumber + 1), e);
-                }
+                // check if there is a next row to read
+                hasNext = readRow();
 
                 return element;
             }

Modified: trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java
===================================================================
--- trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java	2011-12-01 15:22:54 UTC (rev 2249)
+++ trunk/nuiton-csv/src/main/java/org/nuiton/util/csv/ModelBuilder.java	2011-12-07 19:51:32 UTC (rev 2250)
@@ -28,8 +28,8 @@
 import java.util.LinkedList;
 
 /**
- * TODO
  *
+ *
  * @author bleny <leny@codelutin.com>
  * @author tchemit <chemit@codelutin.com>
  * @since 2.4

Modified: trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java
===================================================================
--- trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java	2011-12-01 15:22:54 UTC (rev 2249)
+++ trunk/nuiton-csv/src/test/java/org/nuiton/util/csv/ImportTest.java	2011-12-07 19:51:32 UTC (rev 2250)
@@ -1,6 +1,9 @@
 /*
  * #%L
  * Nuiton Utils :: Nuiton Csv
+ * 
+ * $Id$
+ * $HeadURL$
  * %%
  * Copyright (C) 2011 CodeLutin
  * %%
@@ -24,11 +27,11 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 import org.nuiton.util.DateUtil;
 import org.nuiton.util.csv.Common.BeanProperty;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.text.DateFormat;
 import java.text.ParseException;
@@ -42,25 +45,50 @@
 /**
  * Created on 28/11/11
  *
- * @author fdesbois <florian.desbois@wiztivi.com>
+ * @author fdesbois <desbois@codelutin.com>
+ * @author tchemit <chemit@codelutin.com>
+ * @since 2.4
  */
 public class ImportTest {
 
-    private DateFormat dateFormat;
+    private DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
 
-    @Before
-    public void setUp() {
-        dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+    /**
+     * Test with {@link SimpleImportModel} that directly implements necessary {@link ImportableColumn}
+     * with a empty file (says the file has only the header definition but no row
+     *
+     * @throws Exception for errors
+     */
+    @Test
+    public void testSimpleImportWithEmptyFile() throws Exception {
+        ImportModel<Row> model = new SimpleImportModel(dateFormat);
+        List<Row> rows = importFile(model, "emptyImportTest.csv");
+        Assert.assertEquals(0, rows.size());
     }
 
     /**
+     * Test with {@link SimpleImportModel} that directly implements necessary {@link ImportableColumn}
+     * with a empty file (says the file has only the header definition but no row
+     *
+     * @throws Exception for errors
+     */
+    @Test
+    public void testSimpleImportWithEmptyFile2() throws Exception {
+        ImportModel<Row> model = new SimpleImportModel(dateFormat);
+        List<Row> rows = importFile(model, "emptyImport2Test.csv");
+        Assert.assertEquals(0, rows.size());
+    }
+
+    /**
      * Test with {@link SimpleImportModel} that directly implements necessary {@link ImportableColumn}.
      *
      * @throws Exception for errors
      */
     @Test
     public void testSimpleImport() throws Exception {
-        execute(new SimpleImportModel(dateFormat));
+        ImportModel<Row> model = new SimpleImportModel(dateFormat);
+        List<Row> rows = importFile(model, "importTest.csv");
+        checkImportTestResult(rows);
     }
 
     /**
@@ -70,40 +98,45 @@
      */
     @Test
     public void testColumnImport() throws Exception {
-        execute(new ColumnImportModel(dateFormat));
+        ImportModel<Row> model = new ColumnImportModel(dateFormat);
+        List<Row> rows = importFile(model, "importTest.csv");
+        checkImportTestResult(rows);
     }
 
-    private void execute(ImportModel<Row> model) {
-
-        InputStream csvStream = getClass().getResourceAsStream("importTest.csv");
-        Import<Row> rowImport = Import.newImport(model, csvStream);
+    protected List<Row> importFile(ImportModel<Row> model, String filePath) throws IOException {
+        InputStream csvStream = getClass().getResourceAsStream(filePath);
         try {
-            Iterator<Row> it = rowImport.iterator();
-            List<Row> rows = newList(it);
-            Assert.assertEquals(3, rows.size());
-            Row row1 = rows.get(0);
-            Assert.assertEquals(DateUtil.createDate(5, 12, 2011), row1.getDate());
-            Assert.assertEquals(new Integer(18), row1.getNumber());
-            Assert.assertEquals("1ère ligne", row1.getTitle());
-            Row row2 = rows.get(1);
-            Assert.assertEquals(DateUtil.createDate(6, 12, 2011), row2.getDate());
-            Assert.assertEquals(new Integer(19), row2.getNumber());
-            Assert.assertEquals("2ème ligne", row2.getTitle());
-            Row row3 = rows.get(2);
-            Assert.assertEquals(DateUtil.createDate(7, 12, 2011), row3.getDate());
-            Assert.assertEquals(new Integer(21), row3.getNumber());
-            Assert.assertEquals("3ème ligne", row3.getTitle());
+            Import<Row> rowImport = Import.newImport(model, csvStream);
+            try {
+                Iterator<Row> it = rowImport.iterator();
+                List<Row> result = new ArrayList<Row>();
+                while (it.hasNext()) {
+                    result.add(it.next());
+                }
+                return result;
+            } finally {
+                rowImport.close();
+            }
         } finally {
-            rowImport.close();
+            csvStream.close();
         }
     }
 
-    private static <T> List<T> newList(Iterator<T> iterator) {
-        List<T> result = new ArrayList<T>();
-        while (iterator.hasNext()) {
-            result.add(iterator.next());
-        }
-        return result;
+    private void checkImportTestResult(List<Row> rows) {
+
+        Assert.assertEquals(3, rows.size());
+        Row row1 = rows.get(0);
+        Assert.assertEquals(DateUtil.createDate(5, 12, 2011), row1.getDate());
+        Assert.assertEquals(new Integer(18), row1.getNumber());
+        Assert.assertEquals("1ère ligne", row1.getTitle());
+        Row row2 = rows.get(1);
+        Assert.assertEquals(DateUtil.createDate(6, 12, 2011), row2.getDate());
+        Assert.assertEquals(new Integer(19), row2.getNumber());
+        Assert.assertEquals("2ème ligne", row2.getTitle());
+        Row row3 = rows.get(2);
+        Assert.assertEquals(DateUtil.createDate(7, 12, 2011), row3.getDate());
+        Assert.assertEquals(new Integer(21), row3.getNumber());
+        Assert.assertEquals("3ème ligne", row3.getTitle());
     }
 
     public static class Row {
@@ -234,7 +267,9 @@
 
         @Override
         public void pushCsvHeaderNames(List<String> headerNames) {
-            log.info("Headers are : " + headerNames);
+            if (log.isDebugEnabled()) {
+                log.debug("Headers are : " + headerNames);
+            }
         }
 
         @Override

Added: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv
===================================================================
--- trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv	                        (rev 0)
+++ trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv	2011-12-07 19:51:32 UTC (rev 2250)
@@ -0,0 +1 @@
+DATE;NUMBER;TITLE


Property changes on: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImport2Test.csv
___________________________________________________________________
Added: svn:keywords
   + Author Date Id Revision HeadURL
Added: svn:eol-style
   + native

Added: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv
===================================================================
--- trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv	                        (rev 0)
+++ trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv	2011-12-07 19:51:32 UTC (rev 2250)
@@ -0,0 +1 @@
+DATE;NUMBER;TITLE
\ No newline at end of file


Property changes on: trunk/nuiton-csv/src/test/resources/org/nuiton/util/csv/emptyImportTest.csv
___________________________________________________________________
Added: svn:keywords
   + Author Date Id Revision HeadURL
Added: svn:eol-style
   + native

    

[Nuiton-utils-commits] r2250 - in trunk/nuiton-csv/src: main/java/org/nuiton/util/csv test/java/org/nuiton/util/csv test/resources/org/nuiton/util/csv

tchemit＠users.nuiton.org