Author: tchemit Date: 2012-03-15 12:37:29 +0100 (Thu, 15 Mar 2012) New Revision: 362 Url: http://forge.codelutin.com/repositories/revision/echobase/362 Log: continue result imports + add more import test dbs Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic.h2.db.gz trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches.h2.db.gz Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticResultByEchotypeImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/CsvModelUtil.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AcousticDataImportServiceIT.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsImportServiceIT.java trunk/echobase-services/src/test/resources/import-data/echobase-0.3-commonData-result.h2.db.gz trunk/echobase-services/src/test/resources/import-data/echobase-0.3-nodata.h2.db.gz trunk/echobase-services/src/test/resources/import-data/echobase-0.3-operation-result.h2.db.gz Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticResultByEchotypeImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticResultByEchotypeImportModel.java 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticResultByEchotypeImportModel.java 2012-03-15 11:37:29 UTC (rev 362) @@ -32,8 +32,13 @@ import org.nuiton.util.csv.ImportRuntimeException; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import static org.nuiton.i18n.I18n.l_; + /** * Model to import {@link Result} for esdu cell and echotypes. * @@ -42,21 +47,32 @@ */ public class AcousticResultByEchotypeImportModel extends CsvModelUtil.AbstractImportModel<AcousticResultByEchotypeImportModelRow> { - public static List<DataMetadata> getMetas(Map<String, DataMetadata> dataMetadataMap, + public static final Pattern REMOVE_DOUBLE_QUOTES_PATTERN = Pattern.compile("\"(.+)\""); + + public static List<DataMetadata> getMetas(Locale locale, + Map<String, DataMetadata> dataMetadataMap, String... headers) { List<String> metadataNames = Lists.newArrayList(headers); metadataNames.remove(AcousticResultByEchotypeImportModelRow.PROPERTY_ECHOTYPE); + metadataNames.remove("\"" + AcousticResultByEchotypeImportModelRow.PROPERTY_ECHOTYPE + "\""); metadataNames.remove("name"); + metadataNames.remove("\"name\""); List<DataMetadata> result = Lists.newArrayList(); - for (String header : headers) { + for (String metadataName : metadataNames) { - DataMetadata dataMetadata = dataMetadataMap.get(header); + Matcher matcher = REMOVE_DOUBLE_QUOTES_PATTERN.matcher(metadataName); + if (matcher.matches()) { + metadataName = matcher.group(1); + } + DataMetadata dataMetadata = dataMetadataMap.get(metadataName); if (dataMetadata == null) { throw new ImportRuntimeException( - "Could not find dataMetadata with name [" + header + "]"); + l_(locale, "echobase.importError.dataMetadata.notFound", + metadataName, + dataMetadataMap.keySet())); } result.add(dataMetadata); } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/CsvModelUtil.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/CsvModelUtil.java 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/CsvModelUtil.java 2012-03-15 11:37:29 UTC (rev 362) @@ -541,7 +541,7 @@ private final Voyage voyage; - private final Pattern ID_PATTERN = Pattern.compile("(.+)[_(.+)]"); + private final Pattern ID_PATTERN = Pattern.compile("(.[^_]+)(_(.+)){0,1}"); private final CellDAO cellDAO; @@ -560,7 +560,7 @@ // must find it Matcher matcher = ID_PATTERN.matcher(value); - if (!matcher.find()) { + if (!matcher.matches()) { throw new ImportRuntimeException( "Can not match cell id pattern with value " + value); } @@ -574,15 +574,15 @@ "Can not found esdu cell [" + esduCellId + "]"); } - if (matcher.groupCount() == 1) { + String elementaryCellId = matcher.group(3); + if (elementaryCellId == null) { + //using a simple esdu cell result = esduCell; } else { // using a elementary cell - String elementaryCellId = matcher.group(2); - result = esduCell.getChildByName(elementaryCellId); if (result == null) { throw new ImportRuntimeException( Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java 2012-03-15 11:37:29 UTC (rev 362) @@ -510,11 +510,17 @@ EchoBaseFunctions.DATA_METADATA_NAME); - Map<String, Echotype> echotypeMap = getEntitiesMap( - Echotype.class, +// Map<String, Echotype> echotypeMap = getEntitiesMap( +// Echotype.class, +// EchoBaseFunctions.ECHOTYPE_NAME); + + Map<String, Echotype> echotypeMap = Maps.uniqueIndex( + voyage.getEchotype(), EchoBaseFunctions.ECHOTYPE_NAME); + List<DataMetadata> metas = - AcousticResultByEchotypeImportModel.getMetas(dataMetadataMap, header); + AcousticResultByEchotypeImportModel.getMetas( + getLocale(), dataMetadataMap, header); CellDAO cellDAO = (CellDAO) getDAO(Cell.class); @@ -597,7 +603,7 @@ EchoBaseFunctions.SPECIES_BARACOUDA_CODE); List<DataMetadata> metas = - AcousticResultByEchotypeImportModel.getMetas(dataMetadataMap, header); + AcousticResultByEchotypeImportModel.getMetas(getLocale(), dataMetadataMap, header); CellDAO cellDAO = (CellDAO) getDAO(Cell.class); Modified: trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties =================================================================== --- trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/main/resources/i18n/echobase-services_fr_FR.properties 2012-03-15 11:37:29 UTC (rev 362) @@ -10,6 +10,7 @@ echobase.common.commonDataImportMode.all=Import Voyage / Transit / Transects echobase.common.commonDataImportMode.transect=Import Transects echobase.common.echotypeFile=Fichier echotype +echobase.importError.dataMetadata.notFound=Une colonne de Metadata nommée %s non trouvée (valeurs existantes : %s) echobase.common.gearMetadataFile=Fichier Méta-données d'engin echobase.common.importDataMode.accousticData=Import des données accoustiques echobase.common.importDataMode.catchesData=Import des données de pêche Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java 2012-03-15 11:37:29 UTC (rev 362) @@ -27,7 +27,10 @@ import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.data.DataAcquisition; import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Echotype; import fr.ifremer.echobase.entities.data.GearMetadataValue; +import fr.ifremer.echobase.entities.data.LengthAgeKey; +import fr.ifremer.echobase.entities.data.LengthWeightKey; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.OperationMetadataValue; import fr.ifremer.echobase.entities.data.Sample; @@ -52,6 +55,12 @@ public static final String IMPORT_DATA_ECHOBASE_OPERATION_RESULT = "/import-data/echobase-0.3-operation-result.h2.db.gz"; + public static final String IMPORT_DATA_ECHOBASE_CATCHES= "/import-data/echobase-0.3-catches.h2.db.gz"; + + public static final String IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC= "/import-data/echobase-0.3-catches-and-acoustic.h2.db.gz"; + + public static final String IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT= "/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz"; + public static final String VESSEL_ID = "fr.ifremer.echobase.entities.references.Vessel#1323196672049#0.9790502711645855"; public static final String MISSION_ID = "fr.ifremer.echobase.entities.references.Mission#1323127544274#0.7939481378378231"; @@ -100,6 +109,10 @@ public static final int NB_ECHOTYPE = 8; + public static final int NB_ESDU_RESULT_BY_ECHOTYPE = 8; + + public static final int NB_ESDU_RESULT_BY_ECHOTYPE_AND_SPECIES= 8; + public void assertImportCommonData() throws TopiaException { assertNbEntities(Voyage.class, NB_VOYAGE); assertNbEntities(Transit.class, NB_TRANSIT); @@ -123,4 +136,11 @@ assertNbEntities(Cell.class, NB_CELL); assertNbEntities(Data.class, NB_DATA); } + + + public void assertImportVoyageResult() throws TopiaException { + assertNbEntities(LengthAgeKey.class, NB_LENGTH_AGE_KEY); + assertNbEntities(LengthWeightKey.class, NB_LENGTH_WEIGHT_KEY); + assertNbEntities(Echotype.class, NB_ECHOTYPE); + } } Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AcousticDataImportServiceIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AcousticDataImportServiceIT.java 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AcousticDataImportServiceIT.java 2012-03-15 11:37:29 UTC (rev 362) @@ -48,15 +48,13 @@ public class AcousticDataImportServiceIT extends AbstractImportDataServiceIT { protected FakeEchoBaseServiceContext initContext() { - return new FakeEchoBaseServiceContext( - IMPORT_DATA_ECHOBASE_COMMON_DATA_RESULT); + return new FakeEchoBaseServiceContext(IMPORT_DATA_ECHOBASE_CATCHES); } protected String[] getImportPath(String filename) { return new String[]{"/import-data", "acousticData", filename}; } - @Ignore @Test public void doImport() throws Exception { @@ -97,8 +95,8 @@ getService(AcousticDataImportService.class); // //FIXME : remove this when db will have such data - service.create(getDAO(DataMetadata.class), DataMetadata.PROPERTY_NAME, "minEIthr"); - service.create(getDAO(DataMetadata.class), DataMetadata.PROPERTY_NAME, "maxEIthr"); +// service.create(getDAO(DataMetadata.class), DataMetadata.PROPERTY_NAME, "minEIthr"); +// service.create(getDAO(DataMetadata.class), DataMetadata.PROPERTY_NAME, "maxEIthr"); // service.create(getDAO(AcousticInstrument.class), AcousticInstrument.PROPERTY_ID, "ES38B"); // service.create(getDAO(AcousticInstrument.class), AcousticInstrument.PROPERTY_ID, "ES70-7C"); // service.create(getDAO(AcousticInstrument.class), AcousticInstrument.PROPERTY_ID, "ES120-7"); Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java (rev 0) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java 2012-03-15 11:37:29 UTC (rev 362) @@ -0,0 +1,119 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services.importdata; + +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.LengthAgeKey; +import fr.ifremer.echobase.entities.data.Result; +import fr.ifremer.echobase.services.FakeEchoBaseServiceContext; +import fr.ifremer.echobase.services.configurations.ResultsImportConfiguration; +import fr.ifremer.echobase.services.configurations.ResultsImportMode; +import fr.ifremer.echobase.services.csv.CsvFileImportResult; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.List; + +/** + * Test import of esdu results. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.3 + */ +public class ResultsEsduImportServiceIT extends AbstractImportDataServiceIT { + + protected FakeEchoBaseServiceContext initContext() { + return new FakeEchoBaseServiceContext( + IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT); + } + + protected String[] getImportPath(String filename) { + return new String[]{"/import-data", "result", filename}; + } + + @Test + public void doImportEsduResultByEchotype() throws Exception { + + assertImportCommonData(); + assertImportOperations(); + assertImportAcousticDatas(); + assertImportVoyageResult(); + + assertNoEntities(Result.class); + + ResultsImportConfiguration conf = + new ResultsImportConfiguration(getLocale()); + + conf.setResultLabel("resultLabel"); + conf.setVoyageId(getVoyageId()); + conf.setImportMode(ResultsImportMode.ACOUSTIC_BY_ECHOTYPE); + + prepareInputFile(conf.getAcousticByEchotypeFile(), + getImportPath("acousticResultByEchotype.csv")); + + ResultsImportService service = + getService(ResultsImportService.class); + + List<CsvFileImportResult> result = + service.doImport(conf, createFakeUser()); + + Assert.assertNotNull(result); + Assert.assertEquals(1, result.size()); + assertCsvImportResult(result.get(0), Result.class, NB_ESDU_RESULT_BY_ECHOTYPE); + } + + @Ignore + @Test + public void doImportEsduResultByEchotypeAndSpecies() throws Exception { + + assertImportCommonData(); + assertImportOperations(); + assertImportAcousticDatas(); + assertImportVoyageResult(); + + assertNoEntities(Result.class); + + ResultsImportConfiguration conf = + new ResultsImportConfiguration(getLocale()); + + conf.setResultLabel("resultLabel"); + conf.setVoyageId(getVoyageId()); + conf.setImportMode(ResultsImportMode.ACOUSTIC_BY_ECHOTYPE); + + prepareInputFile(conf.getAcousticByEchotypeFile(), + getImportPath("acousticResultByEchotypeAndSpecies.csv")); + + ResultsImportService service = + getService(ResultsImportService.class); + + List<CsvFileImportResult> result = + service.doImport(conf, createFakeUser()); + + Assert.assertNotNull(result); + Assert.assertEquals(1, result.size()); + assertCsvImportResult(result.get(0), Result.class, NB_ESDU_RESULT_BY_ECHOTYPE_AND_SPECIES); + } + +} \ No newline at end of file Property changes on: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsImportServiceIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsImportServiceIT.java 2012-03-15 10:41:06 UTC (rev 361) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsImportServiceIT.java 2012-03-15 11:37:29 UTC (rev 362) @@ -31,6 +31,7 @@ import fr.ifremer.echobase.services.configurations.ResultsImportMode; import fr.ifremer.echobase.services.csv.CsvFileImportResult; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.util.List; @@ -45,7 +46,7 @@ protected FakeEchoBaseServiceContext initContext() { return new FakeEchoBaseServiceContext( - IMPORT_DATA_ECHOBASE_COMMON_DATA_RESULT); + IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC); } protected String[] getImportPath(String filename) { @@ -53,6 +54,44 @@ } @Test + public void doImportResultByVoyage() throws Exception { + assertImportCommonData(); + assertNoEntities(LengthAgeKey.class); + assertNoEntities(LengthWeightKey.class); + assertNoEntities(Echotype.class); + + ResultsImportConfiguration conf = + new ResultsImportConfiguration(getLocale()); + + conf.setVoyageId(getVoyageId()); + conf.setImportMode(ResultsImportMode.VOYAGE); + + prepareInputFile(conf.getLengthAgeKeyFile(), + getImportPath("lengthAgeKey.csv")); + prepareInputFile(conf.getLengthWeightKeyFile(), + getImportPath("lengthWeightKey.csv")); + prepareInputFile(conf.getEchotypeFile(), getImportPath("echotype.csv")); + + + ResultsImportService service = + getService(ResultsImportService.class); + + List<CsvFileImportResult> result = + service.doImport(conf, createFakeUser()); + + Assert.assertNotNull(result); + Assert.assertEquals(3, result.size()); + assertCsvImportResult(result.get(0), LengthAgeKey.class, + NB_LENGTH_AGE_KEY); + assertCsvImportResult(result.get(1), LengthWeightKey.class, + NB_LENGTH_WEIGHT_KEY); + assertCsvImportResult(result.get(2), Echotype.class, + NB_ECHOTYPE); + + } + + @Ignore + @Test public void doImportLengthAgeKey() throws Exception { assertImportCommonData(); @@ -79,6 +118,7 @@ NB_LENGTH_AGE_KEY); } + @Ignore @Test public void doImportLengthWeightKey() throws Exception { @@ -106,6 +146,7 @@ NB_LENGTH_WEIGHT_KEY); } + @Ignore @Test public void doImportEchotype() throws Exception { Added: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz =================================================================== (Binary files differ) Property changes on: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: svn:keywords + Author Date Id Revision HeadURL Added: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic.h2.db.gz =================================================================== (Binary files differ) Property changes on: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic.h2.db.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: svn:keywords + Author Date Id Revision HeadURL Added: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches.h2.db.gz =================================================================== (Binary files differ) Property changes on: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches.h2.db.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: svn:keywords + Author Date Id Revision HeadURL Modified: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-commonData-result.h2.db.gz =================================================================== (Binary files differ) Modified: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-nodata.h2.db.gz =================================================================== (Binary files differ) Modified: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-operation-result.h2.db.gz =================================================================== (Binary files differ)