October 2012 - Echobase-commits - mailman.cloud.codelutin.com

r751 - in trunk: . echobase-domain echobase-services echobase-tools echobase-ui
by tchemit＠users.forge.codelutin.com Oct. 29, 2012

Oct. 29, 2012

Author: tchemit Date: 2012-10-29 09:18:41 +0100 (Mon, 29 Oct 2012) New Revision: 751 Url: http://forge.codelutin.com/repositories/revision/echobase/751 Log: upgrade to snapshot version for release + add new release files Modified: trunk/echobase-domain/pom.xml trunk/echobase-services/pom.xml trunk/echobase-tools/pom.xml trunk/echobase-ui/pom.xml trunk/pom.xml Modified: trunk/echobase-domain/pom.xml =================================================================== --- trunk/echobase-domain/pom.xml 2012-10-29 07:42:06 UTC (rev 750) +++ trunk/echobase-domain/pom.xml 2012-10-29 08:18:41 UTC (rev 751) @@ -9,7 +9,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>2.0-rc-1</version> + <version>2.0-rc-1-SNAPSHOT</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/echobase-services/pom.xml =================================================================== --- trunk/echobase-services/pom.xml 2012-10-29 07:42:06 UTC (rev 750) +++ trunk/echobase-services/pom.xml 2012-10-29 08:18:41 UTC (rev 751) @@ -6,7 +6,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>2.0-rc-1</version> + <version>2.0-rc-1-SNAPSHOT</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/echobase-tools/pom.xml =================================================================== --- trunk/echobase-tools/pom.xml 2012-10-29 07:42:06 UTC (rev 750) +++ trunk/echobase-tools/pom.xml 2012-10-29 08:18:41 UTC (rev 751) @@ -11,7 +11,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>2.0-rc-1</version> + <version>2.0-rc-1-SNAPSHOT</version> </parent> <groupId>fr.ifremer.echobase</groupId> @@ -24,8 +24,10 @@ <csvVersion>2012-04-12</csvVersion> <redmine.releaseFiles> + ${basedir}/target/echobase-referentiel-${project.version}.echobase ${basedir}/target/echobase-legacyData-${project.version}.echobase, - ${basedir}/target/echobase-referentiel-${project.version}.echobase + ${basedir}/target/echobase-legacyData-extraCells-1998_2004-${project.version}.echobase, + ${basedir}/target/echobase-legacyData-extraCells-2005_2010-${project.version}.echobase </redmine.releaseFiles> </properties> @@ -226,6 +228,35 @@ </includes> </configuration> </plugin> + + <plugin> +  + <artifactId>maven-antrun-plugin</artifactId> + <executions> + + <execution> + <id>copy-extra-cells</id> + <phase>generate-resources</phase> + <configuration> + <tasks> + <copy verbose="${maven.verbose}" + failonerror="false" + overwrite="true" file="${basedir}/src/test/echobase-2012-04-12/legacy-extraCells-1998_2004.echobase" + toFile="${basedir}/target/echobase-legacyData-extraCells-1998_2004-${project.version}.echobase"> + </copy> + <copy verbose="${maven.verbose}" + failonerror="false" + overwrite="true" file="${basedir}/src/test/echobase-2012-04-12/legacy-extraCells-2005_2010.echobase" + toFile="${basedir}/target/echobase-legacyData-extraCells-2005_2010-${project.version}.echobase"> + </copy> + </tasks> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </profile> Modified: trunk/echobase-ui/pom.xml =================================================================== --- trunk/echobase-ui/pom.xml 2012-10-29 07:42:06 UTC (rev 750) +++ trunk/echobase-ui/pom.xml 2012-10-29 08:18:41 UTC (rev 751) @@ -7,7 +7,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>2.0-rc-1</version> + <version>2.0-rc-1-SNAPSHOT</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2012-10-29 07:42:06 UTC (rev 750) +++ trunk/pom.xml 2012-10-29 08:18:41 UTC (rev 751) @@ -11,7 +11,7 @@ <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>2.0-rc-1</version> + <version>2.0-rc-1-SNAPSHOT</version> <modules> <module>echobase-domain</module>

1 0

r750 - in trunk: . echobase-domain echobase-services echobase-services/src/test/java/fr/ifremer/echobase/services echobase-tools echobase-ui
by tchemit＠users.forge.codelutin.com Oct. 29, 2012

Oct. 29, 2012

Author: tchemit Date: 2012-10-29 08:42:06 +0100 (Mon, 29 Oct 2012) New Revision: 750 Url: http://forge.codelutin.com/repositories/revision/echobase/750 Log: update to version 2.0-rc-1 + add missing headers Modified: trunk/echobase-domain/pom.xml trunk/echobase-services/pom.xml trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java trunk/echobase-tools/pom.xml trunk/echobase-ui/pom.xml trunk/pom.xml Modified: trunk/echobase-domain/pom.xml =================================================================== --- trunk/echobase-domain/pom.xml 2012-10-26 11:45:45 UTC (rev 749) +++ trunk/echobase-domain/pom.xml 2012-10-29 07:42:06 UTC (rev 750) @@ -9,7 +9,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>1.4-SNAPSHOT</version> + <version>2.0-rc-1</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/echobase-services/pom.xml =================================================================== --- trunk/echobase-services/pom.xml 2012-10-26 11:45:45 UTC (rev 749) +++ trunk/echobase-services/pom.xml 2012-10-29 07:42:06 UTC (rev 750) @@ -6,7 +6,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>1.4-SNAPSHOT</version> + <version>2.0-rc-1</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java 2012-10-26 11:45:45 UTC (rev 749) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java 2012-10-29 07:42:06 UTC (rev 750) @@ -1,5 +1,28 @@ package fr.ifremer.echobase.services; +/* + * #%L + * EchoBase :: Services + * $Id$ + * $HeadURL:$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + import com.google.common.collect.Sets; import fr.ifremer.echobase.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.data.Cell; Modified: trunk/echobase-tools/pom.xml =================================================================== --- trunk/echobase-tools/pom.xml 2012-10-26 11:45:45 UTC (rev 749) +++ trunk/echobase-tools/pom.xml 2012-10-29 07:42:06 UTC (rev 750) @@ -11,7 +11,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>1.4-SNAPSHOT</version> + <version>2.0-rc-1</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/echobase-ui/pom.xml =================================================================== --- trunk/echobase-ui/pom.xml 2012-10-26 11:45:45 UTC (rev 749) +++ trunk/echobase-ui/pom.xml 2012-10-29 07:42:06 UTC (rev 750) @@ -7,7 +7,7 @@ <parent> <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>1.4-SNAPSHOT</version> + <version>2.0-rc-1</version> </parent> <groupId>fr.ifremer.echobase</groupId> Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2012-10-26 11:45:45 UTC (rev 749) +++ trunk/pom.xml 2012-10-29 07:42:06 UTC (rev 750) @@ -11,7 +11,7 @@ <groupId>fr.ifremer</groupId> <artifactId>echobase</artifactId> - <version>1.4-SNAPSHOT</version> + <version>2.0-rc-1</version> <modules> <module>echobase-domain</module>

1 0

r749 - trunk
by tchemit＠users.forge.codelutin.com Oct. 26, 2012

Oct. 26, 2012

Author: tchemit Date: 2012-10-26 13:45:45 +0200 (Fri, 26 Oct 2012) New Revision: 749 Url: http://forge.codelutin.com/repositories/revision/echobase/749 Log: Update mavenpom4redmine to 3.4.2. Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2012-10-25 17:17:39 UTC (rev 748) +++ trunk/pom.xml 2012-10-26 11:45:45 UTC (rev 749) @@ -6,7 +6,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>3.4</version> + <version>3.4.2</version> </parent> <groupId>fr.ifremer</groupId>

1 0

r748 - in trunk/echobase-tools/src/test/echobase-2012-04-12/csv: association baracouda
by tchemit＠users.forge.codelutin.com Oct. 25, 2012

Oct. 25, 2012

Author: tchemit Date: 2012-10-25 19:17:39 +0200 (Thu, 25 Oct 2012) New Revision: 748 Url: http://forge.codelutin.com/repositories/revision/echobase/748 Log: #1594: Certaines cellules esdu ne sont pas import?\195?\169es (revert data files) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_childs.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_data.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/DataProcessing_cell.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Cell.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Data.csv.gz Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_childs.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_data.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/DataProcessing_cell.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Cell.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Data.csv.gz =================================================================== (Binary files differ)

1 0

r747 - trunk/echobase-tools/src/test/echobase-2012-04-12
by tchemit＠users.forge.codelutin.com Oct. 25, 2012

Oct. 25, 2012

Author: tchemit Date: 2012-10-25 19:07:38 +0200 (Thu, 25 Oct 2012) New Revision: 747 Url: http://forge.codelutin.com/repositories/revision/echobase/747 Log: #1594: Certaines cellules esdu ne sont pas import?\195?\169es (use split files to do large imports) Added: trunk/echobase-tools/src/test/echobase-2012-04-12/legacy-extraCells-1998_2004.echobase trunk/echobase-tools/src/test/echobase-2012-04-12/legacy-extraCells-2005_2010.echobase Added: trunk/echobase-tools/src/test/echobase-2012-04-12/legacy-extraCells-1998_2004.echobase =================================================================== (Binary files differ) Property changes on: trunk/echobase-tools/src/test/echobase-2012-04-12/legacy-extraCells-1998_2004.echobase ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: trunk/echobase-tools/src/test/echobase-2012-04-12/legacy-extraCells-2005_2010.echobase =================================================================== (Binary files differ) Property changes on: trunk/echobase-tools/src/test/echobase-2012-04-12/legacy-extraCells-2005_2010.echobase ___________________________________________________________________ Added: svn:mime-type + application/octet-stream

1 0

r746 - trunk/echobase-services/src/test/java/fr/ifremer/echobase/services
by tchemit＠users.forge.codelutin.com Oct. 25, 2012

Oct. 25, 2012

Author: tchemit Date: 2012-10-25 15:49:16 +0200 (Thu, 25 Oct 2012) New Revision: 746 Url: http://forge.codelutin.com/repositories/revision/echobase/746 Log: revert it Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java 2012-10-25 13:40:55 UTC (rev 745) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java 2012-10-25 13:49:16 UTC (rev 746) @@ -143,13 +143,13 @@ AcousticImportService service = getService(AcousticImportService.class); - AcousticImportService.POSSIBLE_YEARS.clear(); +// AcousticImportService.POSSIBLE_YEARS.clear(); +// +// for (int year : years) { +// +// AcousticImportService.POSSIBLE_YEARS.add(year); +// } - for (int year : years) { - - AcousticImportService.POSSIBLE_YEARS.add(year); - } - service.doImport(conf, createFakeUser()); System.gc();

1 0

r745 - in trunk/echobase-tools/src/test/echobase-2012-04-12/csv: association baracouda
by tchemit＠users.forge.codelutin.com Oct. 25, 2012

Oct. 25, 2012

Author: tchemit Date: 2012-10-25 15:40:55 +0200 (Thu, 25 Oct 2012) New Revision: 745 Url: http://forge.codelutin.com/repositories/revision/echobase/745 Log: refs #1594: Certaines cellules esdu ne sont pas import?\195?\169es (ajout des cellules + data manquantes) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_childs.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_data.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/DataProcessing_cell.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Cell.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Data.csv.gz Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_childs.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Cell_data.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/DataProcessing_cell.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Cell.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/Data.csv.gz =================================================================== (Binary files differ)

1 0

r744 - in trunk/echobase-tools/src/test/echobase-2012-04-12/csv: association baracouda
by tchemit＠users.forge.codelutin.com Oct. 25, 2012

Oct. 25, 2012

Author: tchemit Date: 2012-10-25 14:56:21 +0200 (Thu, 25 Oct 2012) New Revision: 744 Url: http://forge.codelutin.com/repositories/revision/echobase/744 Log: refs #1594: Certaines cellules esdu ne sont pas import?\195?\169es (ajout des dataAcquisition/dataProcessing manquant pour les ann?\195?\169es 2003-2004) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/DataAcquisition_dataProcessing.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Transect_dataAcquisition.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/DataAcquisition.csv.gz trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/DataProcessing.csv.gz Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/DataAcquisition_dataProcessing.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/association/Transect_dataAcquisition.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/DataAcquisition.csv.gz =================================================================== (Binary files differ) Modified: trunk/echobase-tools/src/test/echobase-2012-04-12/csv/baracouda/DataProcessing.csv.gz =================================================================== (Binary files differ)

1 0

r743 - in trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata: . csv
by tchemit＠users.forge.codelutin.com Oct. 25, 2012

Oct. 25, 2012

Author: tchemit Date: 2012-10-25 14:45:08 +0200 (Thu, 25 Oct 2012) New Revision: 743 Url: http://forge.codelutin.com/repositories/revision/echobase/743 Log: back to normal acoustic import Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-24 12:18:36 UTC (rev 742) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-25 12:45:08 UTC (rev 743) @@ -25,9 +25,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.echobase.EchoBaseFunctions; +import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; @@ -50,6 +50,7 @@ import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.importdata.csv.AcousticImportModel; import fr.ifremer.echobase.services.importdata.csv.AcousticImportRow; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.csv.Import; @@ -58,15 +59,14 @@ import java.io.Reader; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import static org.nuiton.i18n.I18n.l_; + /** * Service to launch a "acoustic data" import. * @@ -79,21 +79,19 @@ private static final Log log = LogFactory.getLog(AcousticImportService.class); - public static final Set<Integer> POSSIBLE_YEARS = Sets.newHashSet(); - @Override protected void startImport( AcousticImportConfiguration configuration, EchoBaseUser user) throws ImportException { -// // get selected voyage -// Voyage voyage = getEntityById(Voyage.class, -// configuration.getVoyageId()); -// -// // get selected vessel -// Vessel vessel = getEntityById(Vessel.class, -// configuration.getVesselId()); + // get selected voyage + Voyage voyage = getEntityById(Voyage.class, + configuration.getVoyageId()); + // get selected vessel + Vessel vessel = getEntityById(Vessel.class, + configuration.getVesselId()); + CellType esduCellType = getEntityByProperty( CellType.class, CellType.PROPERTY_ID, "Esdu"); Preconditions.checkNotNull(esduCellType); @@ -104,8 +102,8 @@ EchoBaseCsvFileImportResult importResult = importMoviesFile(configuration, - null,//voyage, - null,//vessel, + voyage, + vessel, esduCellType, elementaryCellType); configuration.addResult(importResult); @@ -137,24 +135,6 @@ return result; } - public Transit getTransit(List<Transit> transits, Date startTime) { - - Preconditions.checkNotNull(startTime); - Transit result = null; - for (Transit t : transits) { - Date transitEndTime = t.getEndTime(); - if (transitEndTime.before(startTime)) { - // transit before required range - continue; - } - - // ok transit contains required range - result = t; - break; - } - return result; - } - private EchoBaseCsvFileImportResult importMoviesFile(AcousticImportConfiguration configuration, Voyage voyage, Vessel vessel, @@ -182,23 +162,15 @@ AcousticImportModel csvModel = new AcousticImportModel( getCsvSeparator(), instrumentsById, dataQualityMap); -// DataAcquisitionDAO dataAcquisitionDAO = -// getDAO(DataAcquisition.class, DataAcquisitionDAO.class); -// DataProcessingDAO dataProcessingDAO = -// getDAO(DataProcessing.class, DataProcessingDAO.class); + DataAcquisitionDAO dataAcquisitionDAO = + getDAO(DataAcquisition.class, DataAcquisitionDAO.class); + DataProcessingDAO dataProcessingDAO = + getDAO(DataProcessing.class, DataProcessingDAO.class); CellDAO cellDAO = getDAO(Cell.class, CellDAO.class); DataDAO dataDAO = getDAO(Data.class, DataDAO.class); - List<Transit> allTransits = getEntities(Transit.class); - Collections.sort(allTransits, new Comparator<Transit>() { - @Override - public int compare(Transit o1, Transit o2) { - return o1.getStartTime().compareTo(o2.getStartTime()); - } - }); + boolean addDataAcquisition = configuration.isAddDataAcquisition(); -// boolean addDataAcquisition = configuration.isAddDataAcquisition(); - String suffix = configuration.getCellPositionReference().getMetadataNameSuffix(); @@ -207,11 +179,8 @@ Reader reader = getInputFileReader(inputFile); Locale locale = getLocale(); String dataProcessingId = null; - Calendar instance = Calendar.getInstance(); try { - Set<String> badTransectIds = Sets.newHashSet(); - Import<AcousticImportRow> importer = Import.newImport(csvModel, reader); DataAcquisition dataAcquisition = null; @@ -224,9 +193,6 @@ Set<String> createdDataAcquisitions = Sets.newHashSet(); - Map<Integer, Integer> years = Maps.newHashMap(); - - for (AcousticImportRow row : importer) { doFlushTransaction(++rowNumber, inputFile, configuration); @@ -240,170 +206,140 @@ } Date startDate = row.getCellDateStart(); - instance.setTime(startDate); - int year = instance.get(Calendar.YEAR); - if (!POSSIBLE_YEARS.contains(year)) { - continue; - } - Date endDate = row.getCellDateEnd(); // get transect to use + Transit transit = voyage.getTransit(startDate); + if (transit == null) { -// Transit transit = voyage.getTransit(startDate); -// if (transit == null) { -// -// // can not find correct transit -// throw new ImportException( -// l_(locale, "echobase.importError.transit.notfound", -// voyage.getName(), -// startDate, -// rowNumber)); -// } -// -// Transect transect = transit.getTransect(vessel); -// if (transect == null) { -// -// // can not find correct transect -// throw new ImportException( -// l_(locale, "echobase.importError.transect.notfound", -// voyage.getName(), -// startDate, -// vessel.getName(), -// rowNumber)); -// } -// -// -// AcousticInstrument instrument = row.getAcousticInstrument(); -// boolean isME70 = -// EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); -// -// if (dataAcquisition == null || -// !instrument.equals(dataAcquisition.getAcousticInstrument())) { -// -// // need to use another data acquisition -// -// if (log.isDebugEnabled()) { -// log.debug("[row " + rowNumber + -// "] New instrument to use (" + -// instrument.getId() + ")"); -// } -// -// if (CollectionUtils.isNotEmpty(elementaryCells)) { -// -// // means miss a esdu cell -// // can not find correct transect -// throw new ImportException( -// l_(locale, "echobase.importError.esduCell.notfound", -// voyage.getName(), -// vessel.getName(), -// instrument.getId(), -// rowNumber)); -// } -// -// String softwareVersion = -// getSoftwareVersion(configuration, isME70); -// -// String soundSpeedCalculations = -// getSoundSpeedCalculations(configuration, isME70); -// -// // try to get existing data acquisition -// dataAcquisition = getDataAcquisition( -// transect, -// instrument, -// addDataAcquisition -// ); -// -// if (dataAcquisition == null) { -// -// // need to create the data acquisition -// dataAcquisition = createDataAcquisition( -// configuration, -// instrument, -// softwareVersion, -// soundSpeedCalculations, -// row, -// dataAcquisitionDAO -// ); -// -// // collect id of the import -// importResult.addId(EchoBaseEntityEnum.DataAcquisition, -// dataAcquisition); -// createdDataAcquisitions.add( -// dataAcquisition.getTopiaId()); -// -// // add dataAcquisition to transect -// transect.addDataAcquisition(dataAcquisition); -// -// if (log.isDebugEnabled()) { -// log.debug("[row " + rowNumber + -// "] New dataAquisition to use (number: " + -// importResult.getNumberCreated( -// EchoBaseEntityEnum.DataAcquisition) + ")"); -// } -// -// // create data processing -// dataProcessing = createDataProcessing( -// configuration, -// dataProcessingId, -// softwareVersion, -// soundSpeedCalculations, -// row, -// dataProcessingDAO -// ); -// -// if (log.isDebugEnabled()) { -// log.debug("[row " + rowNumber + -// "] New dataProcessing to use (" + -// dataProcessing.getId() + ")"); -// } -// -// importResult.incrementsNumberCreated( -// EchoBaseEntityEnum.DataProcessing); -// -// // add it to data acquisition -// dataAcquisition.addDataProcessing(dataProcessing); -// } -// } -// -// if (!row.isCellToAdd()) { -// -// // dead cell not to be imported -// if (log.isDebugEnabled()) { -// log.debug("Will not import cell of row " + rowNumber); -// } -// continue; -// } + // can not find correct transit + throw new ImportException( + l_(locale, "echobase.importError.transit.notfound", + voyage.getName(), + startDate, + rowNumber)); + } - int cellType = row.getCellType(); + Transect transect = transit.getTransect(vessel); + if (transect == null) { - String esduCellId = cellDateFormat.format(endDate); + // can not find correct transect + throw new ImportException( + l_(locale, "echobase.importError.transect.notfound", + voyage.getName(), + startDate, + vessel.getName(), + rowNumber)); + } - DataQuality dataQuality = row.getDataQuality(); + AcousticInstrument instrument = row.getAcousticInstrument(); + boolean isME70 = + EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); - Transit transit = getTransit(allTransits, startDate); - Transect transect = transit.getTransect().iterator().next(); - if (transect.isDataAcquisitionEmpty()) { + if (dataAcquisition == null || + !instrument.equals(dataAcquisition.getAcousticInstrument())) { - // no dataAcquisition - if (!badTransectIds.contains(transect.getTopiaId())) { - badTransectIds.add(transect.getTopiaId()); - if (log.isErrorEnabled()) { - log.error("Could not find dataAcquision for transit " + transit.getStartTime() + " - " + transect.getTitle()); + // need to use another data acquisition + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + + "] New instrument to use (" + + instrument.getId() + ")"); + } + + if (CollectionUtils.isNotEmpty(elementaryCells)) { + + // means miss a esdu cell + // can not find correct transect + throw new ImportException( + l_(locale, "echobase.importError.esduCell.notfound", + voyage.getName(), + vessel.getName(), + instrument.getId(), + rowNumber)); + } + + String softwareVersion = + getSoftwareVersion(configuration, isME70); + + String soundSpeedCalculations = + getSoundSpeedCalculations(configuration, isME70); + + // try to get existing data acquisition + dataAcquisition = getDataAcquisition( + transect, + instrument, + addDataAcquisition + ); + + if (dataAcquisition == null) { + + // need to create the data acquisition + dataAcquisition = createDataAcquisition( + configuration, + instrument, + softwareVersion, + soundSpeedCalculations, + row, + dataAcquisitionDAO + ); + + // collect id of the import + importResult.addId(EchoBaseEntityEnum.DataAcquisition, + dataAcquisition); + createdDataAcquisitions.add( + dataAcquisition.getTopiaId()); + + // add dataAcquisition to transect + transect.addDataAcquisition(dataAcquisition); + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + + "] New dataAquisition to use (number: " + + importResult.getNumberCreated( + EchoBaseEntityEnum.DataAcquisition) + ")"); } + + // create data processing + dataProcessing = createDataProcessing( + configuration, + dataProcessingId, + softwareVersion, + soundSpeedCalculations, + row, + dataProcessingDAO + ); + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + + "] New dataProcessing to use (" + + dataProcessing.getId() + ")"); + } + + importResult.incrementsNumberCreated( + EchoBaseEntityEnum.DataProcessing); + + // add it to data acquisition + dataAcquisition.addDataProcessing(dataProcessing); } + } + + if (!row.isCellToAdd()) { + + // dead cell not to be imported + if (log.isDebugEnabled()) { + log.debug("Will not import cell of row " + rowNumber); + } continue; } - Integer nb = years.get(year); - if (nb == null) { - nb = 0; - } - nb++; - years.put(year, nb); - dataAcquisition = transect.getDataAcquisition().iterator().next(); - dataProcessing = dataAcquisition.getDataProcessing().iterator().next(); + int cellType = row.getCellType(); + String esduCellId = cellDateFormat.format(endDate); + + DataQuality dataQuality = row.getDataQuality(); + + // collect Esdu cellIds only if dataAcquisitions was not created here boolean collectCellIds = !createdDataAcquisitions.contains( dataAcquisition.getTopiaId()); @@ -490,19 +426,6 @@ } } - List<Integer> sortedYears = Lists.newArrayList(years.keySet()); - Collections.sort(sortedYears); - for (Integer year : sortedYears) { - if (log.isInfoEnabled()) { - log.info("Seens years " + year + " :: " + years.get(year)); - } - } - for (String badTransectId : badTransectIds) { - if (log.isInfoEnabled()) { - log.info("Missing dataAcquisition for transect " + badTransectId); - } - } - return importResult; } catch (ImportRuntimeException e) { throw new ImportException(locale, inputFile, e); @@ -533,10 +456,8 @@ // create Depth data dataValue = row.getEsduCellDataDepth(); - if (dataValue != null) { createCellData(dao, cell, dataMetadatas, "DepthRefSurface" + suffix, dataValue, dataQuality, importResult); - } // create Time Start data dataValue = cellDateFormat.format(row.getCellDateStart()); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-24 12:18:36 UTC (rev 742) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-25 12:45:08 UTC (rev 743) @@ -62,13 +62,13 @@ newIgnoredColumn("MOVIES_EILayer\\sndset\\mainBeamAthwartSteeringAngle");//L newMandatoryColumn("MOVIES_EILayer\\sndset\\absorptionCoef", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_ABSORPTION, EchoBaseCsvUtil.FLOAT);//M newMandatoryColumn("MOVIES_EILayer\\sndset\\transmissionPower", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_POWER, EchoBaseCsvUtil.INTEGER);//N - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//O - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//P + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.FLOAT);//O + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.FLOAT);//P newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAlong");//Q newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAthwart");//R - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//S - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//T - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//U + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.FLOAT);//S + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.FLOAT);//T + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.FLOAT);//U newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinDepth");//V newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMaxDepth");//W newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinLevel");//X @@ -94,7 +94,7 @@ newIgnoredColumn("MOVIES_EILayer\\shipnav\\roll");//AR newIgnoredColumn("MOVIES_EILayer\\shipnav\\pitch");//AS newIgnoredColumn("MOVIES_EILayer\\shipnav\\heave");//AT - //newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU + newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU newIgnoredColumn("MOVIES_EILayer\\shipnav\\draught");//AV // Cell elementary Datas @@ -105,8 +105,8 @@ newMandatoryColumn("MOVIES_EILayer\\cellset\\depthend", AcousticImportRow.PROPERTY_CELL_DEPTH_END, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BA newIgnoredColumn("MOVIES_EILayer\\cellset\\indexstart");//BB newIgnoredColumn("MOVIES_EILayer\\cellset\\indexend");//BC - newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BD - newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BE + newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BD + newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BE newMandatoryColumn("MOVIES_EILayer\\cellset\\lat", AcousticImportRow.PROPERTY_CELL_LATITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BF newMandatoryColumn("MOVIES_EILayer\\cellset\\long", AcousticImportRow.PROPERTY_CELL_LONGITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BG @@ -119,8 +119,8 @@ newIgnoredColumn("MOVIES_EILayer\\eilayer");//BN newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticImportRow.PROPERTY_CELL_NASC, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER);//BO newIgnoredColumn("MOVIES_EILayer\\eilayer\\sv");//BP - //newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ - //newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR + newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ + newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR newIgnoredColumn("MOVIES_EILayer\\boterr");//BR newIgnoredColumn("MOVIES_EILayer\\boterr\\sa");//BT newIgnoredColumn("MOVIES_EILayer\\boterr\\ni");//BU

1 0

r742 - in trunk: echobase-domain/src/main/java/fr/ifremer/echobase/csv echobase-services/src/main/java/fr/ifremer/echobase/services/importdata echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv echobase-services/src/test/java/fr/ifremer/echobase/services
by tchemit＠users.forge.codelutin.com Oct. 24, 2012

Oct. 24, 2012

Author: tchemit Date: 2012-10-24 14:18:36 +0200 (Wed, 24 Oct 2012) New Revision: 742 Url: http://forge.codelutin.com/repositories/revision/echobase/742 Log: add test to fix cells Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java 2012-10-24 12:18:36 UTC (rev 742) @@ -47,7 +47,9 @@ public static final String CELLULE_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSS"; public static final ValueParser<Date> IMPORT_DAY_TIME_ECHOBASE = new DateValue(CELLULE_DATE_FORMAT); + public static final ValueParser<Date> IMPORT_DAY_TIME_ECHOBASE2 = new DateValue("dd/MM/yyyy HH:mm:ss.SSSS"); + public static final ValueParser<Boolean> INT_TO_BOOLEAN_PARSER = new ValueParser<Boolean>() { @Override @@ -69,6 +71,21 @@ }; + public static final ValueParser<Float> NA_TO_PRIMITIVE_FLOAT_PARSER = new FloatParserFormatter(0f, false) { + + @Override + protected Float parseNoneEmptyValue(String value) { + Float result = null; + if (!"NA".equals(value)) { + result = super.parseNoneEmptyValue(value); + } else { + result = defaultValue; + } + return result; + } + + }; + public static final ValueParser<Integer> NA_TO_INTEGER_PARSER = new IntegerParserFormatter(null, true) { @Override Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java 2012-10-24 12:18:36 UTC (rev 742) @@ -112,7 +112,7 @@ s0 = TIME_LOG.log(s0, "importDone"); // add result in log book and compute resume to show in result - String result = computeResultAndLogBookEntry(configuration, user); +// String result = computeResultAndLogBookEntry(configuration, user); // do commit commitTransaction("Could not execute import " + @@ -120,7 +120,8 @@ TIME_LOG.log(s0, "importCommited"); - return result; +// return result; + return ""; } catch (EchoBaseTechnicalException e) { throw new ImportException(e.getMessage(), e); } @@ -287,6 +288,8 @@ return message; } + static long s0 =TimeLog.getTime(); + protected final void doFlushTransaction(int rowNumber, InputFile inputFile, M configuration) { @@ -294,10 +297,13 @@ if (rowNumber % 1000 == 0) { // flush each 1000 imported rows String message = getImportMessage(configuration, inputFile); - long s0 = TimeLog.getTime(); - flushTransaction("Could not flush transaction " + message); - TIME_LOG.log(s0, message); - +// long s0 = TimeLog.getTime(); +// flushTransaction("Could not flush transaction " + message); + s0 = TIME_LOG.log(s0, message); + commitTransaction("Could committransaction " + message); + if (log.isInfoEnabled()) { + log.info("Progression, row done: " + rowNumber); + } } } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-24 12:18:36 UTC (rev 742) @@ -25,9 +25,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.echobase.EchoBaseFunctions; -import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; @@ -50,7 +50,6 @@ import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.importdata.csv.AcousticImportModel; import fr.ifremer.echobase.services.importdata.csv.AcousticImportRow; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.csv.Import; @@ -59,14 +58,15 @@ import java.io.Reader; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import static org.nuiton.i18n.I18n.l_; - /** * Service to launch a "acoustic data" import. * @@ -79,19 +79,21 @@ private static final Log log = LogFactory.getLog(AcousticImportService.class); + public static final Set<Integer> POSSIBLE_YEARS = Sets.newHashSet(); + @Override protected void startImport( AcousticImportConfiguration configuration, EchoBaseUser user) throws ImportException { - // get selected voyage - Voyage voyage = getEntityById(Voyage.class, - configuration.getVoyageId()); +// // get selected voyage +// Voyage voyage = getEntityById(Voyage.class, +// configuration.getVoyageId()); +// +// // get selected vessel +// Vessel vessel = getEntityById(Vessel.class, +// configuration.getVesselId()); - // get selected vessel - Vessel vessel = getEntityById(Vessel.class, - configuration.getVesselId()); - CellType esduCellType = getEntityByProperty( CellType.class, CellType.PROPERTY_ID, "Esdu"); Preconditions.checkNotNull(esduCellType); @@ -100,11 +102,12 @@ CellType.class, CellType.PROPERTY_ID, "Elementary"); Preconditions.checkNotNull(elementaryCellType); - EchoBaseCsvFileImportResult importResult = importMoviesFile(configuration, - voyage, - vessel, - esduCellType, - elementaryCellType); + EchoBaseCsvFileImportResult importResult = + importMoviesFile(configuration, + null,//voyage, + null,//vessel, + esduCellType, + elementaryCellType); configuration.addResult(importResult); } @@ -134,12 +137,29 @@ return result; } + public Transit getTransit(List<Transit> transits, Date startTime) { + + Preconditions.checkNotNull(startTime); + Transit result = null; + for (Transit t : transits) { + Date transitEndTime = t.getEndTime(); + if (transitEndTime.before(startTime)) { + // transit before required range + continue; + } + + // ok transit contains required range + result = t; + break; + } + return result; + } + private EchoBaseCsvFileImportResult importMoviesFile(AcousticImportConfiguration configuration, Voyage voyage, Vessel vessel, CellType esduCellType, - CellType elementaryCellType - ) throws ImportException { + CellType elementaryCellType) throws ImportException { InputFile inputFile = configuration.getMoviesFile(); @@ -162,15 +182,23 @@ AcousticImportModel csvModel = new AcousticImportModel( getCsvSeparator(), instrumentsById, dataQualityMap); - DataAcquisitionDAO dataAcquisitionDAO = - getDAO(DataAcquisition.class, DataAcquisitionDAO.class); - DataProcessingDAO dataProcessingDAO = - getDAO(DataProcessing.class, DataProcessingDAO.class); +// DataAcquisitionDAO dataAcquisitionDAO = +// getDAO(DataAcquisition.class, DataAcquisitionDAO.class); +// DataProcessingDAO dataProcessingDAO = +// getDAO(DataProcessing.class, DataProcessingDAO.class); CellDAO cellDAO = getDAO(Cell.class, CellDAO.class); DataDAO dataDAO = getDAO(Data.class, DataDAO.class); - boolean addDataAcquisition = configuration.isAddDataAcquisition(); + List<Transit> allTransits = getEntities(Transit.class); + Collections.sort(allTransits, new Comparator<Transit>() { + @Override + public int compare(Transit o1, Transit o2) { + return o1.getStartTime().compareTo(o2.getStartTime()); + } + }); +// boolean addDataAcquisition = configuration.isAddDataAcquisition(); + String suffix = configuration.getCellPositionReference().getMetadataNameSuffix(); @@ -179,8 +207,11 @@ Reader reader = getInputFileReader(inputFile); Locale locale = getLocale(); String dataProcessingId = null; + Calendar instance = Calendar.getInstance(); try { + Set<String> badTransectIds = Sets.newHashSet(); + Import<AcousticImportRow> importer = Import.newImport(csvModel, reader); DataAcquisition dataAcquisition = null; @@ -193,6 +224,9 @@ Set<String> createdDataAcquisitions = Sets.newHashSet(); + Map<Integer, Integer> years = Maps.newHashMap(); + + for (AcousticImportRow row : importer) { doFlushTransaction(++rowNumber, inputFile, configuration); @@ -206,140 +240,170 @@ } Date startDate = row.getCellDateStart(); + instance.setTime(startDate); + int year = instance.get(Calendar.YEAR); + if (!POSSIBLE_YEARS.contains(year)) { + continue; + } + Date endDate = row.getCellDateEnd(); // get transect to use - Transit transit = voyage.getTransit(startDate); - if (transit == null) { - // can not find correct transit - throw new ImportException( - l_(locale, "echobase.importError.transit.notfound", - voyage.getName(), - startDate, - rowNumber)); - } +// Transit transit = voyage.getTransit(startDate); +// if (transit == null) { +// +// // can not find correct transit +// throw new ImportException( +// l_(locale, "echobase.importError.transit.notfound", +// voyage.getName(), +// startDate, +// rowNumber)); +// } +// +// Transect transect = transit.getTransect(vessel); +// if (transect == null) { +// +// // can not find correct transect +// throw new ImportException( +// l_(locale, "echobase.importError.transect.notfound", +// voyage.getName(), +// startDate, +// vessel.getName(), +// rowNumber)); +// } +// +// +// AcousticInstrument instrument = row.getAcousticInstrument(); +// boolean isME70 = +// EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); +// +// if (dataAcquisition == null || +// !instrument.equals(dataAcquisition.getAcousticInstrument())) { +// +// // need to use another data acquisition +// +// if (log.isDebugEnabled()) { +// log.debug("[row " + rowNumber + +// "] New instrument to use (" + +// instrument.getId() + ")"); +// } +// +// if (CollectionUtils.isNotEmpty(elementaryCells)) { +// +// // means miss a esdu cell +// // can not find correct transect +// throw new ImportException( +// l_(locale, "echobase.importError.esduCell.notfound", +// voyage.getName(), +// vessel.getName(), +// instrument.getId(), +// rowNumber)); +// } +// +// String softwareVersion = +// getSoftwareVersion(configuration, isME70); +// +// String soundSpeedCalculations = +// getSoundSpeedCalculations(configuration, isME70); +// +// // try to get existing data acquisition +// dataAcquisition = getDataAcquisition( +// transect, +// instrument, +// addDataAcquisition +// ); +// +// if (dataAcquisition == null) { +// +// // need to create the data acquisition +// dataAcquisition = createDataAcquisition( +// configuration, +// instrument, +// softwareVersion, +// soundSpeedCalculations, +// row, +// dataAcquisitionDAO +// ); +// +// // collect id of the import +// importResult.addId(EchoBaseEntityEnum.DataAcquisition, +// dataAcquisition); +// createdDataAcquisitions.add( +// dataAcquisition.getTopiaId()); +// +// // add dataAcquisition to transect +// transect.addDataAcquisition(dataAcquisition); +// +// if (log.isDebugEnabled()) { +// log.debug("[row " + rowNumber + +// "] New dataAquisition to use (number: " + +// importResult.getNumberCreated( +// EchoBaseEntityEnum.DataAcquisition) + ")"); +// } +// +// // create data processing +// dataProcessing = createDataProcessing( +// configuration, +// dataProcessingId, +// softwareVersion, +// soundSpeedCalculations, +// row, +// dataProcessingDAO +// ); +// +// if (log.isDebugEnabled()) { +// log.debug("[row " + rowNumber + +// "] New dataProcessing to use (" + +// dataProcessing.getId() + ")"); +// } +// +// importResult.incrementsNumberCreated( +// EchoBaseEntityEnum.DataProcessing); +// +// // add it to data acquisition +// dataAcquisition.addDataProcessing(dataProcessing); +// } +// } +// +// if (!row.isCellToAdd()) { +// +// // dead cell not to be imported +// if (log.isDebugEnabled()) { +// log.debug("Will not import cell of row " + rowNumber); +// } +// continue; +// } - Transect transect = transit.getTransect(vessel); - if (transect == null) { + int cellType = row.getCellType(); - // can not find correct transect - throw new ImportException( - l_(locale, "echobase.importError.transect.notfound", - voyage.getName(), - startDate, - vessel.getName(), - rowNumber)); - } + String esduCellId = cellDateFormat.format(endDate); - AcousticInstrument instrument = row.getAcousticInstrument(); - boolean isME70 = - EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); + DataQuality dataQuality = row.getDataQuality(); - if (dataAcquisition == null || - !instrument.equals(dataAcquisition.getAcousticInstrument())) { + Transit transit = getTransit(allTransits, startDate); + Transect transect = transit.getTransect().iterator().next(); + if (transect.isDataAcquisitionEmpty()) { - // need to use another data acquisition - - if (log.isDebugEnabled()) { - log.debug("[row " + rowNumber + - "] New instrument to use (" + - instrument.getId() + ")"); - } - - if (CollectionUtils.isNotEmpty(elementaryCells)) { - - // means miss a esdu cell - // can not find correct transect - throw new ImportException( - l_(locale, "echobase.importError.esduCell.notfound", - voyage.getName(), - vessel.getName(), - instrument.getId(), - rowNumber)); - } - - String softwareVersion = - getSoftwareVersion(configuration, isME70); - - String soundSpeedCalculations = - getSoundSpeedCalculations(configuration, isME70); - - // try to get existing data acquisition - dataAcquisition = getDataAcquisition( - transect, - instrument, - addDataAcquisition - ); - - if (dataAcquisition == null) { - - // need to create the data acquisition - dataAcquisition = createDataAcquisition( - configuration, - instrument, - softwareVersion, - soundSpeedCalculations, - row, - dataAcquisitionDAO - ); - - // collect id of the import - importResult.addId(EchoBaseEntityEnum.DataAcquisition, - dataAcquisition); - createdDataAcquisitions.add( - dataAcquisition.getTopiaId()); - - // add dataAcquisition to transect - transect.addDataAcquisition(dataAcquisition); - - if (log.isDebugEnabled()) { - log.debug("[row " + rowNumber + - "] New dataAquisition to use (number: " + - importResult.getNumberCreated( - EchoBaseEntityEnum.DataAcquisition) + ")"); + // no dataAcquisition + if (!badTransectIds.contains(transect.getTopiaId())) { + badTransectIds.add(transect.getTopiaId()); + if (log.isErrorEnabled()) { + log.error("Could not find dataAcquision for transit " + transit.getStartTime() + " - " + transect.getTitle()); } - - // create data processing - dataProcessing = createDataProcessing( - configuration, - dataProcessingId, - softwareVersion, - soundSpeedCalculations, - row, - dataProcessingDAO - ); - - if (log.isDebugEnabled()) { - log.debug("[row " + rowNumber + - "] New dataProcessing to use (" + - dataProcessing.getId() + ")"); - } - - importResult.incrementsNumberCreated( - EchoBaseEntityEnum.DataProcessing); - - // add it to data acquisition - dataAcquisition.addDataProcessing(dataProcessing); } - } - - if (!row.isCellToAdd()) { - - // dead cell not to be imported - if (log.isDebugEnabled()) { - log.debug("Will not import cell of row " + rowNumber); - } continue; } + Integer nb = years.get(year); + if (nb == null) { + nb = 0; + } + nb++; + years.put(year, nb); - int cellType = row.getCellType(); + dataAcquisition = transect.getDataAcquisition().iterator().next(); + dataProcessing = dataAcquisition.getDataProcessing().iterator().next(); - String esduCellId = cellDateFormat.format(endDate); - - DataQuality dataQuality = row.getDataQuality(); - - // collect Esdu cellIds only if dataAcquisitions was not created here boolean collectCellIds = !createdDataAcquisitions.contains( dataAcquisition.getTopiaId()); @@ -426,6 +490,19 @@ } } + List<Integer> sortedYears = Lists.newArrayList(years.keySet()); + Collections.sort(sortedYears); + for (Integer year : sortedYears) { + if (log.isInfoEnabled()) { + log.info("Seens years " + year + " :: " + years.get(year)); + } + } + for (String badTransectId : badTransectIds) { + if (log.isInfoEnabled()) { + log.info("Missing dataAcquisition for transect " + badTransectId); + } + } + return importResult; } catch (ImportRuntimeException e) { throw new ImportException(locale, inputFile, e); @@ -456,8 +533,10 @@ // create Depth data dataValue = row.getEsduCellDataDepth(); - createCellData(dao, cell, dataMetadatas, "DepthRefSurface" + suffix, - dataValue, dataQuality, importResult); + if (dataValue != null) { + createCellData(dao, cell, dataMetadatas, "DepthRefSurface" + suffix, + dataValue, dataQuality, importResult); + } // create Time Start data dataValue = cellDateFormat.format(row.getCellDateStart()); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-24 12:18:36 UTC (rev 742) @@ -62,13 +62,13 @@ newIgnoredColumn("MOVIES_EILayer\\sndset\\mainBeamAthwartSteeringAngle");//L newMandatoryColumn("MOVIES_EILayer\\sndset\\absorptionCoef", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_ABSORPTION, EchoBaseCsvUtil.FLOAT);//M newMandatoryColumn("MOVIES_EILayer\\sndset\\transmissionPower", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_POWER, EchoBaseCsvUtil.INTEGER);//N - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.FLOAT);//O - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.FLOAT);//P + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//O + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//P newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAlong");//Q newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAthwart");//R - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.FLOAT);//S - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.FLOAT);//T - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.FLOAT);//U + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//S + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//T + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//U newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinDepth");//V newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMaxDepth");//W newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinLevel");//X @@ -94,7 +94,7 @@ newIgnoredColumn("MOVIES_EILayer\\shipnav\\roll");//AR newIgnoredColumn("MOVIES_EILayer\\shipnav\\pitch");//AS newIgnoredColumn("MOVIES_EILayer\\shipnav\\heave");//AT - newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU + //newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU newIgnoredColumn("MOVIES_EILayer\\shipnav\\draught");//AV // Cell elementary Datas @@ -105,8 +105,8 @@ newMandatoryColumn("MOVIES_EILayer\\cellset\\depthend", AcousticImportRow.PROPERTY_CELL_DEPTH_END, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BA newIgnoredColumn("MOVIES_EILayer\\cellset\\indexstart");//BB newIgnoredColumn("MOVIES_EILayer\\cellset\\indexend");//BC - newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BD - newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BE + newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BD + newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BE newMandatoryColumn("MOVIES_EILayer\\cellset\\lat", AcousticImportRow.PROPERTY_CELL_LATITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BF newMandatoryColumn("MOVIES_EILayer\\cellset\\long", AcousticImportRow.PROPERTY_CELL_LONGITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BG @@ -119,8 +119,8 @@ newIgnoredColumn("MOVIES_EILayer\\eilayer");//BN newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticImportRow.PROPERTY_CELL_NASC, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER);//BO newIgnoredColumn("MOVIES_EILayer\\eilayer\\sv");//BP - newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ - newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR + //newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ + //newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR newIgnoredColumn("MOVIES_EILayer\\boterr");//BR newIgnoredColumn("MOVIES_EILayer\\boterr\\sa");//BT newIgnoredColumn("MOVIES_EILayer\\boterr\\ni");//BU Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java (rev 0) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java 2012-10-24 12:18:36 UTC (rev 742) @@ -0,0 +1,263 @@ +package fr.ifremer.echobase.services; + +import com.google.common.collect.Sets; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.services.importdata.AcousticImportConfiguration; +import fr.ifremer.echobase.services.importdata.AcousticImportService; +import fr.ifremer.echobase.services.importdata.CellPositionReference; +import fr.ifremer.echobase.services.importdata.ImportException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.TopiaDAO; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Set; + +/** + * To fix cells and datas. + * <p/> + * see http://forge.codelutin.com/issues/1592 + * see http://forge.codelutin.com/issues/1593 + * + * @author tchemit <chemit(a)codelutin.com> + * @since 1.4 + */ +public class FixCellsIT extends EchoBaseTestServiceSupport { + + /** Logger. */ + private static final Log log = LogFactory.getLog(FixCellsIT.class); + + public static int[] YEARS = { + 1998, + 2000, + 2001, + 2002, + 2003, + 2004, + 2005, + 2006, + 2007, + 2008, + 2009, + 2010 + }; + /* + 1998 :: 10017 + 2000 :: 12075 + 2001 :: 18000 + 2002 :: 15303 + 2003 :: 27315 + 2004 :: 23955 + 2005 :: 14805 + 2006 :: 12825 + 2007 :: 16005 + 2008 :: 20595 + 2009 :: 24690 + 2010 :: 15150 + + */ + + @Override + protected FakeEchoBaseServiceContext initContext() { +// return new FakeEchoBaseServiceContext(null); + return new FakeEchoBaseServiceContext("/fixcells.h2.db.gz"); + } + + Set<String> positionDatas = Sets.newHashSet("LatitudeStart", "LongitudeStart"); + + Set<String> timeStartData = Sets.newHashSet("TimeStart"); + + DateFormat dateInstance = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CEST' yyyy"); + + DateFormat dateInstance2 = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CET' yyyy"); + + DateFormat isoDateFormat = new SimpleDateFormat(EchoBaseCsvUtil.CELLULE_DATE_FORMAT); + + @Test + public void lissage() throws Exception { + +// importdb(ImportDbMode.FREE, "/fixcells.echobase"); + +// importMissingEsduCells(); + +// fixCellDatas(); + + exportDatabase(); + } + + protected void importMissingEsduCells() throws IOException, ImportException { + + AcousticImportConfiguration conf = new AcousticImportConfiguration(getLocale()); + prepareInputFile(conf.getMoviesFile(), "/missingEsduCells.csv.gz"); + conf.setCellPositionReference(CellPositionReference.START); + +// doImport(conf, +// 1998, +// 2000, +// 2001, +// 2002, +// 2003, +// 2004); + /* +Seens years 1998 :: 10017 +Seens years 2000 :: 12075 +Seens years 2001 :: 18000 +Seens years 2002 :: 15303 +Seens years 2003 :: 26595 + + */ + + doImport(conf, + 2005, + 2006, + 2007, + 2008, + 2009, + 2010); + + /* +2005 :: 14805 +2006 :: 12825 +2007 :: 16005 +2008 :: 20595 +2009 :: 24690 +2010 :: 15150 + + */ + + } + + protected void doImport(AcousticImportConfiguration conf, int... years) throws ImportException { + + System.gc(); + + AcousticImportService service = getService(AcousticImportService.class); + + AcousticImportService.POSSIBLE_YEARS.clear(); + + for (int year : years) { + + AcousticImportService.POSSIBLE_YEARS.add(year); + } + + service.doImport(conf, createFakeUser()); + + System.gc(); + } + + protected void fixCellDatas() throws TopiaException, ParseException { + + int nb = 0; + TopiaDAO<Cell> cellDAO = getDAO(Cell.class); + + long nbCells = cellDAO.count(); + for (Cell cell : cellDAO) { + + if (log.isDebugEnabled()) { + log.debug("treat cell " + cell.getName() + " - " + + cell.getCellType().getName()); + } + + if (!cell.isDataEmpty()) { + for (Data data : cell.getData()) { + String name = data.getDataMetadata().getName(); + if (timeStartData.contains(name)) { + fixDateFormat(cell, data); + } else if (positionDatas.contains(name)) { + fixDataPosition(data); + } + } + } + + if (++nb % 10000 == 0) { + if (log.isInfoEnabled()) { + log.info("Flush at " + nb + "/" + nbCells); + } + flushTransaction("Do flush"); + } + } + + commitTransaction("Do commit!"); + } + + private void exportDatabase() throws TopiaException, IOException { + + TopiaDAO<Voyage> voyageDAO = getDAO(Voyage.class); + + Voyage voyage; + if (voyageDAO.count() == 0) { + + // crate voyage + + voyage = voyageDAO.create(Voyage.PROPERTY_MISSION, getEntities(Mission.class).get(0), + Voyage.PROPERTY_NAME, "name", + Voyage.PROPERTY_START_DATE, newDate(), + Voyage.PROPERTY_END_DATE, newDate(), + Voyage.PROPERTY_START_PORT, "ici", + Voyage.PROPERTY_END_PORT, "labas", + Voyage.PROPERTY_DESCRIPTION, "description", + Voyage.PROPERTY_DATUM, "datum" + ); + + commitTransaction("Do commit!"); + } else { + voyage = voyageDAO.findAll().get(0); + } + + commitTransaction("Do commit!"); + + exportDb(voyage, "fixCells"); + + } + + private void fixDateFormat(Cell cell, Data data) throws ParseException { + + String dataValue = data.getDataValue(); + Date oldDate; + try { + oldDate = dateInstance.parse(dataValue); + } catch (ParseException e) { + oldDate = dateInstance2.parse(dataValue); + } + + String newDate = isoDateFormat.format(oldDate); + + if (log.isInfoEnabled()) { + log.info("Treat date " + dataValue + " - to " + newDate); + } + data.setDataValue(newDate); + cell.setName(newDate); + } + + + private void fixDataPosition(Data data) { + + String dataValue = data.getDataValue(); + + double sign = dataValue.endsWith("W") ? -1 : 1; + + double x = Float.valueOf(dataValue.substring(0, dataValue.length() - 1)); + + double x1 = Math.floor(x / 100); + double x2 = Math.floor(x - x1 * 100) / 60; + double x3 = ((x - x1 * 100) - x2 * 60) / 100; + double newValue = sign * (x1 + x2 + x3); + + if (log.isInfoEnabled()) { + log.info("Treat data position " + x + " to " + newValue); + } + + data.setDataValue(String.valueOf(newValue)); + } + +} Property changes on: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native

1 0