[Echobase-commits] r162 - in trunk/echobase-services/src/main/java/fr/ifremer/echobase: . services services/models

11 Dec 2011

Author: tchemit
Date: 2011-12-11 19:15:44 +0100 (Sun, 11 Dec 2011)
New Revision: 162

Url: http://forge.codelutin.com/repositories/revision/echobase/162

Log:
introduce EchoBaseIOUtil

Added:
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/EchoBaseIOUtil.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/ImportExportModelSupport.java
Modified:
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java
   trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationCsvModel.java

Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/EchoBaseIOUtil.java
===================================================================

--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/EchoBaseIOUtil.java	                        (rev 0)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/EchoBaseIOUtil.java	2011-12-11 18:15:44 UTC (rev 162)
@@ -0,0 +1,117 @@
+/*
+ * #%L
+ * EchoBase :: Services
+ * 
+ * $Id$
+ * $HeadURL$
+ * %%
+ * Copyright (C) 2011 Ifremer, Codelutin
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * #L%
+ */
+package fr.ifremer.echobase;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.FileUtils;
+import org.nuiton.util.FileUtil;
+import org.nuiton.util.ZipUtil;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * Usefull methods on io.
+ *
+ * @author tchemit <chemit@codelutin.com>
+ * @since 0.2
+ */
+public class EchoBaseIOUtil {
+
+
+    public static void compressZipFile(File zipFile, File directory) {
+        compressZipFile(zipFile, directory, true);
+    }
+
+    public static void compressZipFile(File zipFile, File directory, boolean delete) {
+        try {
+            ZipUtil.compress(zipFile, directory);
+        } catch (IOException eee) {
+            throw new EchoBaseTechnicalException(
+                    "Can not create zip file " + zipFile, eee);
+        } finally {
+            if (delete) {
+                FileUtil.deleteRecursively(directory);
+            }
+        }
+    }
+
+    public static void copyFile(File destinationFile,
+                                Collection<File> filesToMerge) throws IOException {
+        if (CollectionUtils.isEmpty(filesToMerge)) {
+
+            // nothing to copy
+            return;
+        }
+        if (filesToMerge.size() == 1) {
+
+            // simple copy
+            FileUtils.copyFile(filesToMerge.iterator().next(), destinationFile);
+        } else {
+
+            // must merge all files
+            mergeFiles(destinationFile, filesToMerge);
+        }
+    }
+
+    public static void mergeFiles(File destination,
+                                  Iterable<File> fileToMerges) throws IOException {
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(destination));
+        try {
+            boolean firstLine = false;
+            for (File toMergeFile : fileToMerges) {
+                BufferedReader reader = new BufferedReader(new FileReader(toMergeFile));
+                try {
+                    // pass the first line (header)
+                    String line = reader.readLine();
+                    if (!firstLine) {
+
+                        // add the csv header
+                        writer.write(line);
+
+                        firstLine = true;
+                    }
+                    while ((line = reader.readLine()) != null) {
+                        writer.newLine();
+                        writer.write(line);
+                    }
+                } finally {
+                    reader.close();
+                }
+
+            }
+        } finally {
+            writer.close();
+        }
+    }
+
+    protected EchoBaseIOUtil() {
+    }
+}


Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/EchoBaseIOUtil.java
___________________________________________________________________
Added: svn:keywords
   + Author Date Id Revision HeadURL
Added: svn:eol-style
   + native

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java	2011-12-11 18:15:03 UTC (rev 161)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbEditorService.java	2011-12-11 18:15:44 UTC (rev 162)
@@ -23,7 +23,6 @@
  */
 package fr.ifremer.echobase.services;
 
-import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import fr.ifremer.echobase.EchoBaseTechnicalException;
@@ -38,7 +37,6 @@
 import fr.ifremer.echobase.services.models.CsvModelUtil;
 import fr.ifremer.echobase.services.models.EntityAssociationCsvModel;
 import fr.ifremer.echobase.services.models.EntityCsvModel;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -46,6 +44,7 @@
 import org.nuiton.topia.framework.TopiaQuery;
 import org.nuiton.topia.persistence.TopiaDAO;
 import org.nuiton.topia.persistence.TopiaEntity;
+import org.nuiton.util.TimeLog;
 import org.nuiton.util.beans.BeanMonitor;
 import org.nuiton.util.beans.PropertyDiff;
 import org.nuiton.util.csv.Export;
@@ -299,12 +298,8 @@
 
     public String exportDatas(TableMeta tableMeta) {
 
-        List<TopiaEntity> datas = getEntities(tableMeta, null, null, null, null);
+        Export<TopiaEntity> export = prepareExport(tableMeta);
 
-        ImportExportModel<TopiaEntity> model = buildForExport(tableMeta);
-
-        Export<TopiaEntity> export = Export.newExport(model, datas);
-
         try {
             return export.startExportAsString();
         } catch (Exception eee) {
@@ -312,34 +307,21 @@
         }
     }
 
-    public String exportDatas(AssociationMeta associationMeta) {
-
-        TableMeta tableMeta = getTableMeta(associationMeta.getSource());
-
-        List<TopiaEntity> datas = getEntities(tableMeta, null, null, null, "size(e." + associationMeta.getName() + ") > 0");
-
-        ImportExportModel<TopiaEntity> model = buildForExport(associationMeta);
-
-        Export<TopiaEntity> export = Export.newExport(model, datas);
-
-        try {
-            return export.startExportAsString();
-        } catch (Exception eee) {
-            throw new EchoBaseTechnicalException("Can not export datas", eee);
-        }
-    }
-
     public void exportDatas(TableMeta tableMeta, File file) {
 
         if (log.isInfoEnabled()) {
             log.info("Export table " + tableMeta + " to " + file);
         }
+        long s1 = TimeLog.getTime();
+        Export<TopiaEntity> export = prepareExport(tableMeta);
+        timeLog.log(s1, "prepareExport");
+        long s2 = TimeLog.getTime();
         try {
-            String content = exportDatas(tableMeta);
-            FileUtils.write(file, content, Charsets.UTF_8.name());
+            export.exportToFile(file);
         } catch (Exception eee) {
             throw new EchoBaseTechnicalException("Can not export datas", eee);
         }
+        timeLog.log(s2, "exportToFile");
     }
 
     public void exportDatas(AssociationMeta associationMeta, File file) {
@@ -347,12 +329,17 @@
         if (log.isInfoEnabled()) {
             log.info("Export association " + associationMeta + " to " + file);
         }
+        long s1 = TimeLog.getTime();
+        Export<TopiaEntity> export = prepareExport(associationMeta);
+        timeLog.log(s1, "prepareExport");
+
+        long s2 = TimeLog.getTime();
         try {
-            String content = exportDatas(associationMeta);
-            FileUtils.write(file, content, Charsets.UTF_8.name());
+            export.exportToFile(file);
         } catch (Exception eee) {
             throw new EchoBaseTechnicalException("Can not export datas", eee);
         }
+        timeLog.log(s2, "exportToFile");
     }
 
     public boolean saveEntity(TableMeta tableMeta,
@@ -498,7 +485,7 @@
 
         Map<String, Object> row = Maps.newLinkedHashMap();
 
-        Collection<ExportableColumn<E, Object>> columns =
+        Iterable<ExportableColumn<E, Object>> columns =
                 loadModel.getColumnsForExport();
 
         for (ExportableColumn<E, Object> column : columns) {
@@ -560,6 +547,16 @@
         return model;
     }
 
+    public <E extends TopiaEntity> ImportExportModel<E> buildForExport(MetaFilenameAware meta) {
+        if (meta instanceof AssociationMeta) {
+            return buildForExport((AssociationMeta) meta);
+        }
+        if (meta instanceof TableMeta) {
+            return buildForExport((TableMeta) meta);
+        }
+        throw new IllegalStateException("Can't do a thing with " + meta);
+    }
+
     public <E extends TopiaEntity> ImportExportModel<E> buildForExport(TableMeta meta) {
 
         EntityCsvModel<E> model = EntityCsvModel.newModel(
@@ -568,7 +565,6 @@
                 TopiaEntity.TOPIA_ID
         );
 
-
         for (ColumnMeta columnMeta : meta) {
             String propertyName = columnMeta.getName();
             Class<?> type = columnMeta.getType();
@@ -647,4 +643,21 @@
         }
         return model;
     }
+
+    protected Export<TopiaEntity> prepareExport(TableMeta tableMeta) {
+        List<TopiaEntity> datas = getEntities(tableMeta, null, null, null, null);
+        ImportExportModel<TopiaEntity> model = buildForExport(tableMeta);
+        return Export.newExport(model, datas);
+    }
+
+    public static final TimeLog timeLog = new TimeLog(DbEditorService.class);
+
+    protected Export<TopiaEntity> prepareExport(AssociationMeta associationMeta) {
+
+        TableMeta tableMeta = getTableMeta(associationMeta.getSource());
+        List<TopiaEntity> datas = getEntities(tableMeta, null, null, null, "size(e." + associationMeta.getName() + ") > 0");
+
+        ImportExportModel<TopiaEntity> model = buildForExport(associationMeta);
+        return Export.newExport(model, datas);
+    }
 }

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java	2011-12-11 18:15:03 UTC (rev 161)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportExportService.java	2011-12-11 18:15:44 UTC (rev 162)
@@ -25,6 +25,7 @@
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import fr.ifremer.echobase.EchoBaseIOUtil;
 import fr.ifremer.echobase.EchoBaseTechnicalException;
 import fr.ifremer.echobase.entities.EchoBaseEntityEnum;
 import fr.ifremer.echobase.entities.EchoBaseUser;
@@ -39,6 +40,7 @@
 import org.nuiton.topia.TopiaException;
 import org.nuiton.topia.persistence.TopiaDAO;
 import org.nuiton.topia.persistence.TopiaEntity;
+import org.nuiton.util.StringUtil;
 import org.nuiton.util.csv.Import;
 import org.nuiton.util.csv.ImportExportModel;
 
@@ -81,7 +83,8 @@
 
             List<String> missingEntries = Lists.newArrayList();
 
-            Map<MetaFilenameAware, ZipEntry> entriestoConsume = Maps.newLinkedHashMap();
+            Map<MetaFilenameAware, ZipEntry> entriestoConsume =
+                    Maps.newLinkedHashMap();
 
             // check that all mandatories
             for (MetaFilenameAware entry : entries) {
@@ -97,10 +100,14 @@
 
             if (!missingEntries.isEmpty()) {
 
-                // there is some missing file entries
-                throw new EchoBaseTechnicalException(
-                        "There is some missing mandatory files " +
-                                missingEntries + " in import " + file);
+                if (log.isWarnEnabled()) {
+                    log.warn("There is " + missingEntries.size() + "missing mandatory files " +
+                                     " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false));
+                }
+
+//                throw new EchoBaseTechnicalException(
+//                        "There is some missing mandatory files " +
+//                                missingEntries + " in import " + file);
             }
 
             // consume all found entries
@@ -162,7 +169,7 @@
             }
         }
 
-        compressZipFile(zipFile, tempDirectory);
+        EchoBaseIOUtil.compressZipFile(zipFile, tempDirectory);
 
         return zipFile;
     }
@@ -199,7 +206,6 @@
         }
     }
 
-
     public MetaFilenameAware[] getEntries() {
 
         List<MetaFilenameAware> result = Lists.newArrayList();
@@ -213,7 +219,8 @@
         return getDbEditorService().getTableMeta(tableName);
     }
 
-    protected void addEntries(List<MetaFilenameAware> result, EchoBaseEntityEnum[] types) {
+    protected void addEntries(List<MetaFilenameAware> result,
+                              EchoBaseEntityEnum[] types) {
         for (EchoBaseEntityEnum type : types) {
 
             TableMeta tableMeta = getTableMeta(type);
@@ -233,9 +240,9 @@
     }
 
     protected void importEntities(TableMeta meta,
-                               ImportExportModel<TopiaEntity> csvModel,
-                               CsvImportResult result,
-                               Reader reader) {
+                                  ImportExportModel<TopiaEntity> csvModel,
+                                  CsvImportResult result,
+                                  Reader reader) {
 
         Import<TopiaEntity> importer = Import.newImport(csvModel, reader);
 

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java	2011-12-11 18:15:03 UTC (rev 161)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java	2011-12-11 18:15:44 UTC (rev 162)
@@ -35,11 +35,7 @@
 import org.nuiton.topia.TopiaException;
 import org.nuiton.topia.persistence.TopiaDAO;
 import org.nuiton.topia.persistence.TopiaEntity;
-import org.nuiton.util.FileUtil;
-import org.nuiton.util.ZipUtil;
 
-import java.io.File;
-import java.io.IOException;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
@@ -129,15 +125,4 @@
         return result;
     }
 
-    protected void compressZipFile(File zipFile, File directory) {
-        try {
-            ZipUtil.compress(zipFile, directory);
-        } catch (IOException eee) {
-            throw new EchoBaseTechnicalException(
-                    "Can not create zip file " + zipFile, eee);
-        } finally {
-            FileUtil.deleteRecursively(directory);
-        }
-    }
-
 }

Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationCsvModel.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationCsvModel.java	2011-12-11 18:15:03 UTC (rev 161)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/EntityAssociationCsvModel.java	2011-12-11 18:15:44 UTC (rev 162)
@@ -58,13 +58,13 @@
     }
 
     @Override
-    public Collection<ExportableColumn<E, Object>> getColumnsForExport() {
+    public Iterable<ExportableColumn<E, Object>> getColumnsForExport() {
         return (Collection)
                 modelBuilder.getColumnsForExport();
     }
 
     @Override
-    public Collection<ImportableColumn<E, Object>> getColumnsForImport() {
+    public Iterable<ImportableColumn<E, Object>> getColumnsForImport() {
         return (Collection)
                 modelBuilder.getColumnsForImport();
     }

Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/ImportExportModelSupport.java
===================================================================
--- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/ImportExportModelSupport.java	                        (rev 0)
+++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/ImportExportModelSupport.java	2011-12-11 18:15:44 UTC (rev 162)
@@ -0,0 +1,71 @@
+/*
+ * #%L
+ * EchoBase :: Services
+ * 
+ * $Id$
+ * $HeadURL$
+ * %%
+ * Copyright (C) 2011 Ifremer, Codelutin
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * #L%
+ */
+package fr.ifremer.echobase.services.models;
+
+import org.nuiton.util.csv.ExportableColumn;
+import org.nuiton.util.csv.ImportExportModel;
+import org.nuiton.util.csv.ImportableColumn;
+import org.nuiton.util.csv.ModelBuilder;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * TODO
+ *
+ * @author tchemit <chemit@codelutin.com>
+ * @since TODO
+ */
+public abstract class ImportExportModelSupport<E> implements ImportExportModel<E> {
+
+    protected char separator;
+
+    protected ModelBuilder<E> modelBuilder;
+
+    protected ImportExportModelSupport(char separator) {
+        this.separator = separator;
+        modelBuilder = new ModelBuilder<E>();
+    }
+
+    @Override
+    public char getSeparator() {
+        return separator;
+    }
+
+    @Override
+    public void pushCsvHeaderNames(List<String> headerNames) {
+    }
+
+    @Override
+    public Collection<ExportableColumn<E, Object>> getColumnsForExport() {
+        return (Collection)
+                modelBuilder.getColumnsForExport();
+    }
+
+    @Override
+    public Collection<ImportableColumn<E, Object>> getColumnsForImport() {
+        return (Collection)
+                modelBuilder.getColumnsForImport();
+    }
+}


Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/models/ImportExportModelSupport.java
___________________________________________________________________
Added: svn:keywords
   + Author Date Id Revision HeadURL
Added: svn:eol-style
   + native

    

[Echobase-commits] r162 - in trunk/echobase-services/src/main/java/fr/ifremer/echobase: . services services/models

tchemit＠users.forge.codelutin.com