Author: tchemit Date: 2011-12-16 02:46:20 +0100 (Fri, 16 Dec 2011) New Revision: 198 Url: http://forge.codelutin.com/repositories/revision/echobase/198 Log: Import des Operation + Sample Added: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoader.java trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoaderIT.java trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/OperationLoaderIT.java trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoaderIT.java trunk/echobase-tools/src/test/resources/csv/new/input-AgeCategory.csv trunk/echobase-tools/src/test/resources/csv/new/input-SpeciesCategory.csv Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java trunk/echobase-entities/src/main/xmi/echobase.zargo trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AbstractLoaderIT.java trunk/src/doc/reunions/reunion-2011-12-14.txt Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java 2011-12-15 21:53:41 UTC (rev 197) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java 2011-12-16 01:46:20 UTC (rev 198) @@ -25,6 +25,7 @@ import com.google.common.base.Function; import fr.ifremer.echobase.entities.data.DataProcessing; +import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.AcousticInstrument; import fr.ifremer.echobase.entities.references.AreaOfOperation; @@ -33,6 +34,7 @@ import fr.ifremer.echobase.entities.references.DataType; import fr.ifremer.echobase.entities.references.DepthStratum; import fr.ifremer.echobase.entities.references.EchotypeCategory; +import fr.ifremer.echobase.entities.references.Gear; import fr.ifremer.echobase.entities.references.Mission; import fr.ifremer.echobase.entities.references.OperationEvent; import fr.ifremer.echobase.entities.references.ReferenceDatum; @@ -40,8 +42,10 @@ import fr.ifremer.echobase.entities.references.SampleDataType; import fr.ifremer.echobase.entities.references.Species; import fr.ifremer.echobase.entities.references.Vessel; +import org.apache.commons.lang3.StringUtils; import org.nuiton.topia.persistence.TopiaEntity; +import javax.annotation.Nullable; import java.io.File; import java.util.Map; @@ -174,6 +178,20 @@ return input.getName(); } }; + public static final Function<Gear, String> GEAR_CASINO_GEAR_NAME = new Function<Gear, String>() { + @Override + public String apply(Gear input) { + //FIXME : Remove the dummy test when all gear in db have a casino name + String casinoGearName = input.getCasinoGearName(); + return StringUtils.isEmpty(casinoGearName) ? input.getName() : casinoGearName; + } + }; + public static final Function<Operation, String> OPERATION_ID = new Function<Operation, String>() { + @Override + public String apply(Operation input) { + return input.getId(); + } + }; public static Function<Map<String, Object>, String> newRowFunction(final String key) { Modified: trunk/echobase-entities/src/main/xmi/echobase.zargo =================================================================== (Binary files differ) Added: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoader.java (rev 0) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoader.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,56 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import fr.ifremer.echobase.entities.meta.TableMeta; +import fr.ifremer.echobase.entities.references.AgeCategory; +import fr.ifremer.echobase.services.models.EntityCsvModel; +import fr.ifremer.echobase.tools.FileType; + +/** + * To load {@link AgeCategory} entities. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class AgeCategoryLoader extends EntityLoader<AgeCategory> { + + public AgeCategoryLoader() { + super(AgeCategory.class, FileType.NEW); + } + + + @Override + protected EntityCsvModel<AgeCategory> createCsvImportModel(TableMeta meta) { + + EntityCsvModel<AgeCategory> model = EntityCsvModel.newModel( + getCsvSeparator(), + meta + ); + model.newMandatoryColumn(AgeCategory.PROPERTY_NAME); + model.newMandatoryColumn(AgeCategory.PROPERTY_MEANING); + return model; + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoader.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java 2011-12-15 21:53:41 UTC (rev 197) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java 2011-12-16 01:46:20 UTC (rev 198) @@ -118,8 +118,7 @@ super.exportDatas(outputDir, targetType); // also export Data - TableMeta dataMeta = getTable(EchoBaseEntityEnum.Data); - exportData(outputDir, targetType, dataMeta); + exportData(outputDir, targetType, getTable(EchoBaseEntityEnum.Data)); } @Override @@ -127,14 +126,12 @@ FileType targetType) throws IOException { super.exportAssociations(outputDir, targetType); - AssociationMeta associationMeta; - // also export cells associations - associationMeta = getMeta().getAssociations(Cell.PROPERTY_CHILDS); - exportAssociation(outputDir, targetType, associationMeta); + exportAssociation(outputDir, targetType, + getMeta().getAssociations(Cell.PROPERTY_CHILDS)); - associationMeta = getMeta().getAssociations(Cell.PROPERTY_DATA); - exportAssociation(outputDir, targetType, associationMeta); + exportAssociation(outputDir, targetType, + getMeta().getAssociations(Cell.PROPERTY_DATA)); } @Override Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java 2011-12-15 21:53:41 UTC (rev 197) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java 2011-12-16 01:46:20 UTC (rev 198) @@ -400,22 +400,10 @@ return universe; } -// protected <E> E getFK(Map<String, E> universe, String fk) { -// return getFK(universe, fk, true); -// } - protected <E> E getFK(Map<String, E> universe, String fk) { -// E entity = universe.get(lowerCase ? fk.toLowerCase() : fk); E entity = universe.get(fk); -// if (strict) - Preconditions.checkNotNull(entity, "Could not find entity with id " + fk); -// else { -// if (entity == null) { -// if (log.isWarnEnabled()) { -// log.warn("Could not find entity with id " + fk); -// } -// } -// } + Preconditions.checkNotNull(entity, + "Could not find entity with id " + fk); return entity; } Added: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java (rev 0) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,350 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.healthmarketscience.jackcess.Database; +import fr.ifremer.echobase.EchoBaseFunctions; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; +import fr.ifremer.echobase.entities.data.GearMetadataValue; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.OperationMetadataValue; +import fr.ifremer.echobase.entities.data.Transect; +import fr.ifremer.echobase.entities.data.Transit; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.meta.TableMeta; +import fr.ifremer.echobase.entities.references.DepthStratum; +import fr.ifremer.echobase.entities.references.Gear; +import fr.ifremer.echobase.entities.references.GearMetadata; +import fr.ifremer.echobase.entities.references.GearMetadataDAO; +import fr.ifremer.echobase.entities.references.OperationEvent; +import fr.ifremer.echobase.entities.references.OperationMetadata; +import fr.ifremer.echobase.entities.references.OperationMetadataDAO; +import fr.ifremer.echobase.tools.FileType; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.TopiaDAO; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * To load {@link Operation} entities. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class OperationLoader extends EntityLoader<Operation> { + + /** Logger. */ + private static final Log log = + LogFactory.getLog(OperationLoader.class); + + public OperationLoader() { + super(Operation.class, EchoBaseEntityEnum.Transect, + Transect.PROPERTY_OPERATION, new EchoBaseEntityEnum[]{ + EchoBaseEntityEnum.OperationMetadata, + EchoBaseEntityEnum.GearMetadata + }, + FileType.BARACOUDA); + } + + @Override + protected void exportDatas(File outputDir, FileType targetType) throws IOException { + super.exportDatas(outputDir, targetType); + + // also export OperationMetadataValue + exportData(outputDir, targetType, + getTable(EchoBaseEntityEnum.OperationMetadataValue)); + + // also export GearMetadataValue + exportData(outputDir, targetType, + getTable(EchoBaseEntityEnum.GearMetadataValue)); + } + + @Override + protected void exportAssociations(File outputDir, FileType targetType) throws IOException { + super.exportAssociations(outputDir, targetType); + + TableMeta meta = getMeta(); + + // export operation metadata values + exportAssociation(outputDir, targetType, + meta.getAssociations(Operation.PROPERTY_OPERATION_METADATA_VALUE)); + + // export gear metadata values + exportAssociation(outputDir, targetType, + meta.getAssociations(Operation.PROPERTY_GEAR_METADATA_VALUE)); + } + + @Override + protected void copyAndLoad(File outputDir) throws IOException, TopiaException { + + super.copyAndLoad(outputDir); + + copyAndLoadEntities(outputDir, getTable(EchoBaseEntityEnum.OperationMetadataValue), false, FileType.BARACOUDA); + copyAndLoadEntities(outputDir, getTable(EchoBaseEntityEnum.GearMetadataValue), false, FileType.BARACOUDA); + + TableMeta meta = getMeta(); + + copyAndLoadEntities(outputDir, meta.getAssociations(Operation.PROPERTY_OPERATION_METADATA_VALUE), false, FileType.ASSOCIATION); + copyAndLoadEntities(outputDir, meta.getAssociations(Operation.PROPERTY_GEAR_METADATA_VALUE), false, FileType.ASSOCIATION); + + } + + @Override + protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { + + Map<String, Voyage> voyagesByName = getUniverse( + Voyage.class, EchoBaseFunctions.VOYAGE_NAME); + + Map<String, DepthStratum> depthStratumByName = getUniverse( + DepthStratum.class, EchoBaseFunctions.DEPTH_STRATUM_ID); + + Map<String, Gear> gearByCasinoGearName = getUniverse( + Gear.class, EchoBaseFunctions.GEAR_CASINO_GEAR_NAME); + + OperationMetadataDAO operationMetadataDAO = + (OperationMetadataDAO) getDAO(OperationMetadata.class); + + //SONDE + OperationMetadata MeanWaterDepthOperationMeta = + operationMetadataDAO.findByName("MeanWaterDepth"); + Preconditions.checkNotNull(MeanWaterDepthOperationMeta); + //JOUR_NUIT + OperationMetadata DielPeriodOperationMeta = + operationMetadataDAO.findByName("DielPeriod"); + Preconditions.checkNotNull(DielPeriodOperationMeta); + //SONDEDEB + OperationMetadata StartWaterDepthOperationMeta = operationMetadataDAO.findByProperties( + OperationMetadata.PROPERTY_NAME, "WaterDepth", + OperationMetadata.PROPERTY_OPERATION_EVENT + "." + OperationEvent.PROPERTY_NAME, "ShootingStart" + ); + Preconditions.checkNotNull(StartWaterDepthOperationMeta); + //SONDEFIN + OperationMetadata EndWaterDepthOperationMeta = operationMetadataDAO.findByProperties( + OperationMetadata.PROPERTY_NAME, "WaterDepth", + OperationMetadata.PROPERTY_OPERATION_EVENT + "." + OperationEvent.PROPERTY_NAME, "HaulingEnd" + ); + Preconditions.checkNotNull(EndWaterDepthOperationMeta); + + GearMetadataDAO gearMetadataDAO = + (GearMetadataDAO) getDAO(GearMetadata.class); + + //LFUNES + GearMetadata CableLengthGearMeta = gearMetadataDAO.findByType("CableLength"); + Preconditions.checkNotNull(CableLengthGearMeta); + + //VMIN + GearMetadata MinSpeedGearMeta = gearMetadataDAO.findByType("MinSpeed"); + Preconditions.checkNotNull(MinSpeedGearMeta); + + //VMAX + GearMetadata MaxSpeedGearMeta = gearMetadataDAO.findByType("MaxSpeed"); + Preconditions.checkNotNull(MaxSpeedGearMeta); + + TopiaDAO<Operation> dao = getDAO(); + TopiaDAO<OperationMetadataValue> operationMetadataValueDAO = + getDAO(OperationMetadataValue.class); + + TopiaDAO<GearMetadataValue> gearMetadataValueDAO = + getDAO(GearMetadataValue.class); + + Multimap<String, Map<String, Object>> stationByCampagneName = Multimaps.index( + db.getTable("STATIONS"), EchoBaseFunctions.newRowFunction("CAMPAGNE")); + + List<Operation> operations = Lists.newArrayList(); + + List<String> messages = Lists.newArrayList(); + + for (String voyageName : stationByCampagneName.keySet()) { + Voyage voyage = voyagesByName.get(voyageName); + Preconditions.checkNotNull(voyage); + Transit transit = voyage.getTransit().iterator().next(); + Preconditions.checkNotNull(transit); + Transect transect = transit.getTransect().iterator().next(); + Preconditions.checkNotNull(transect); + Collection<Map<String, Object>> rows = + stationByCampagneName.get(voyageName); + for (Map<String, Object> row : rows) { + + String status = (String) row.get("ETAT"); + if ("FICTIF".equalsIgnoreCase(status)) { + + // skip fictif operation + continue; + } + String operationId = (String) row.get("NOSTA"); + + DepthStratum depthStratum = null; + String depthStratumId = (String) row.get("STA_IMAGES"); + if (StringUtils.isBlank(depthStratumId) || + "nul".equals(depthStratumId)) { + messages.add("No depthStraum defined"); + } else { + depthStratum = getFK(depthStratumByName, depthStratumId.trim()); + } + + String gearId = (String) row.get("ENGIN"); + if (gearId == null) { + messages.add("No gear id"); + } else { + Gear gear = getFK(gearByCasinoGearName, gearId); + + Float shootingStartLatitude = (Float) row.get("LATDEB"); + if (shootingStartLatitude == null) { + messages.add("No shootingStartLatitude"); + shootingStartLatitude = 0f; + } + Float shootingEndLatitude = (Float) row.get("LATDD"); + if (shootingEndLatitude == null) { + messages.add("No shootingEndLatitude"); + shootingEndLatitude = 0f; + } + Float shootingStartLongitude = (Float) row.get("LGDEB"); + if (shootingStartLongitude == null) { + messages.add("No shootingStartLongitude"); + shootingStartLongitude = 0f; + } + Float shootingEndLongitude = (Float) row.get("LGDD"); + if (shootingEndLongitude == null) { + messages.add("No shootingEndLongitude"); + shootingEndLongitude = 0f; + } + + Date shootingStartTime = (Date) row.get("DHTUDEB"); + Date shootingEndTime = (Date) row.get("DHTUFIN"); + + String comment = (String) row.get("COMMENTAIRE"); + + // create a operation + Operation operation = dao.create( + Operation.PROPERTY_ID, operationId, + Operation.PROPERTY_DEPTH_STRATUM, depthStratum, + Operation.PROPERTY_GEAR, gear, + Operation.PROPERTY_GEAR_SHOOTING_START_LATITUDE, shootingStartLatitude, + Operation.PROPERTY_GEAR_SHOOTING_END_LATITUDE, shootingEndLatitude, + Operation.PROPERTY_GEAR_SHOOTING_START_LONGITUDE, shootingStartLongitude, + Operation.PROPERTY_GEAR_SHOOTING_END_LONGITUDE, shootingEndLongitude, + Operation.PROPERTY_GEAR_SHOOTING_START_TIME, shootingStartTime, + Operation.PROPERTY_GEAR_SHOOTING_END_TIME, shootingEndTime, + Operation.PROPERTY_COMMENT, comment + ); + + addOperationMetadataValue(operationMetadataValueDAO, + operation, + MeanWaterDepthOperationMeta, + row.get("SONDE")); + + addOperationMetadataValue(operationMetadataValueDAO, + operation, + DielPeriodOperationMeta, + row.get("JOUR_NUIT")); + + addOperationMetadataValue(operationMetadataValueDAO, + operation, + StartWaterDepthOperationMeta, + row.get("SONDEDEB")); + + addOperationMetadataValue(operationMetadataValueDAO, + operation, + EndWaterDepthOperationMeta, + row.get("SONDEFIN")); + + addGearMetadataValue(gearMetadataValueDAO, + operation, + CableLengthGearMeta, + row.get("LFUNES")); + + addGearMetadataValue(gearMetadataValueDAO, + operation, + MinSpeedGearMeta, + row.get("VMIN")); + + addGearMetadataValue(gearMetadataValueDAO, + operation, + MaxSpeedGearMeta, + row.get("VMAX")); + + operations.add(operation); + transect.addOperation(operation); + + } + if (!messages.isEmpty()) { + + if (log.isWarnEnabled()) { + log.warn("For operation " + operationId + ", status " + status + " : " + messages); + } + messages.clear(); + } + } + } + + if (log.isInfoEnabled()) { + log.info("Found " + operations.size() + " operation(s)."); + } + } + + protected void addOperationMetadataValue(TopiaDAO<OperationMetadataValue> dao, + Operation operation, + OperationMetadata meta, + Object value + ) throws TopiaException { + if (value == null) { + return; + } + OperationMetadataValue operationMetadataValue = dao.create( + OperationMetadataValue.PROPERTY_OPERATION_METADATA, meta, + OperationMetadataValue.PROPERTY_DATA_VALUE, value.toString() + ); + operation.addOperationMetadataValue(operationMetadataValue); + } + + protected void addGearMetadataValue(TopiaDAO<GearMetadataValue> dao, + Operation operation, + GearMetadata meta, + Object value + ) throws TopiaException { + + if (value == null) { + return; + } + GearMetadataValue gearMetadataValue = dao.create( + GearMetadataValue.PROPERTY_GEAR_METADATA, meta, + GearMetadataValue.PROPERTY_DATA_VALUE, value.toString(), + GearMetadataValue.PROPERTY_GEAR, operation.getGear() + ); + operation.addGearMetadataValue(gearMetadataValue); + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java (rev 0) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,329 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.healthmarketscience.jackcess.Database; +import fr.ifremer.echobase.EchoBaseFunctions; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; +import fr.ifremer.echobase.entities.data.Operation; +import fr.ifremer.echobase.entities.data.Sample; +import fr.ifremer.echobase.entities.data.SampleData; +import fr.ifremer.echobase.entities.data.SampleDataDAO; +import fr.ifremer.echobase.entities.meta.TableMeta; +import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SampleDataTypeDAO; +import fr.ifremer.echobase.entities.references.SampleType; +import fr.ifremer.echobase.entities.references.SampleTypeDAO; +import fr.ifremer.echobase.entities.references.Species; +import fr.ifremer.echobase.tools.FileType; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.TopiaDAO; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +/** + * To load {@link Sample} entities. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class SampleLoader extends EntityLoader<Sample> { + + /** Logger. */ + private static final Log log = + LogFactory.getLog(SampleLoader.class); + + public SampleLoader() { + super(Sample.class, EchoBaseEntityEnum.Operation, + Operation.PROPERTY_SAMPLE, new EchoBaseEntityEnum[]{ + EchoBaseEntityEnum.SampleDataType, + EchoBaseEntityEnum.SampleType, + }, + FileType.BARACOUDA); + } + + @Override + protected void exportDatas(File outputDir, FileType targetType) throws IOException { + super.exportDatas(outputDir, targetType); + + exportData(outputDir, targetType, + getTable(EchoBaseEntityEnum.SampleData)); + } + + @Override + protected void exportAssociations(File outputDir, FileType targetType) throws IOException { + super.exportAssociations(outputDir, targetType); + + TableMeta meta = getMeta(); + + exportAssociation(outputDir, targetType, + meta.getAssociations(Sample.PROPERTY_SAMPLE_DATA)); + + } + + @Override + protected void copyAndLoad(File outputDir) throws IOException, TopiaException { + + super.copyAndLoad(outputDir); + + copyAndLoadEntities(outputDir, getTable(EchoBaseEntityEnum.SampleData), false, FileType.BARACOUDA); + + TableMeta meta = getMeta(); + + copyAndLoadEntities(outputDir, meta.getAssociations(Sample.PROPERTY_SAMPLE_DATA), false, FileType.ASSOCIATION); + + } + + @Override + protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { + + Map<String, Operation> operationById = getUniverse( + Operation.class, EchoBaseFunctions.OPERATION_ID); + + Map<String, Species> speciesByName = getUniverse( + Species.class, EchoBaseFunctions.SPECIES_BARACOUDA_CODE); + + Multimap<String, Map<String, Object>> mensurationByNosta = Multimaps.index( + db.getTable("MENSURATIONS", true), + EchoBaseFunctions.newRowFunction("NOSTA") + ); + + Multimap<String, Map<String, Object>> capturesByNosta = Multimaps.index( + db.getTable("CAPTURES", true), + EchoBaseFunctions.newRowFunction("NOSTA") + ); + SampleTypeDAO sampleTypeDAO = + (SampleTypeDAO) getDAO(SampleType.class); + + SampleType sampleTypeTotal = sampleTypeDAO.findByName("Total"); + Preconditions.checkNotNull(sampleTypeTotal); + SampleType sampleTypeUnsorted = sampleTypeDAO.findByName("Unsorted"); + Preconditions.checkNotNull(sampleTypeUnsorted); + SampleType sampleTypeSubsample = sampleTypeDAO.findByName("Subsample"); + Preconditions.checkNotNull(sampleTypeSubsample); + + SampleDataTypeDAO sampleDataTypeDAO = + (SampleDataTypeDAO) getDAO(SampleDataType.class); + + SampleDataDAO sampleDataDAO = (SampleDataDAO) getDAO(SampleData.class); + + SampleDataType sampleDataTypeMeanLength = sampleDataTypeDAO.findByName("MeanLengthcm"); + Preconditions.checkNotNull(sampleDataTypeMeanLength); + SampleDataType sampleDataTypeMeanWeight = sampleDataTypeDAO.findByName("MeanWeightg"); + Preconditions.checkNotNull(sampleDataTypeMeanWeight); + SampleDataType sampleDataTypeNoPerKg = sampleDataTypeDAO.findByName("NoPerKg"); + Preconditions.checkNotNull(sampleDataTypeNoPerKg); + SampleDataType sampleDataTypeNumberAtLength = sampleDataTypeDAO.findByName("NumberAtLength"); + Preconditions.checkNotNull(sampleDataTypeNumberAtLength); + SampleDataType sampleDataTypeWeightAtLength = sampleDataTypeDAO.findByName("WeightAtLengthkg"); + Preconditions.checkNotNull(sampleDataTypeWeightAtLength); + + TopiaDAO<Sample> dao = getDAO(); + + int nbSampleUnsorted = 0; + int nbSampleTotal = 0; + int nbSubSample = 0; + + for (String operationId : capturesByNosta.keySet()) { + Preconditions.checkNotNull(operationId); + Operation operation = operationById.get(operationId); + if (operation == null) { + if (log.isWarnEnabled()) { + log.warn("Operation " + operationId + " not found..."); + } + continue; + } + for (Map<String, Object> row : capturesByNosta.get(operationId)) { + String type = (String) row.get("CATEG"); + + SampleType sampleType; + + if ("H".equals(type)) { + sampleType = sampleTypeUnsorted; + nbSampleUnsorted++; + } else { + sampleType = sampleTypeTotal; + nbSampleTotal++; + } + + String speciesId = (String) row.get("GENRE_ESP"); + Species species = getFK(speciesByName, speciesId); + + Integer number = (Integer) row.get("NTOT"); + if (number == null) { + if (log.isWarnEnabled()) { + log.warn("NTOT is null for operation " + operationId + "/" + speciesId); + } + number = 0; + } + Float weight = (Float) row.get("PT_ESP"); + + Sample sample = dao.create( + Sample.PROPERTY_SPECIES, species, + Sample.PROPERTY_SAMPLE_TYPE, sampleType, + Sample.PROPERTY_NUMBER_SAMPLED, number, + Sample.PROPERTY_SAMPLE_WEIGHT, weight + ); + + operation.addSample(sample); + + addSampleData(sampleDataDAO, + sample, + sampleDataTypeMeanWeight, + "NA", + row.get("PMOY")); + + addSampleData(sampleDataDAO, + sample, + sampleDataTypeMeanLength, + "NA", + row.get("LMOY")); + + addSampleData(sampleDataDAO, + sample, + sampleDataTypeNoPerKg, + "NA", + row.get("MOULE_MENS")); + + Collection<Map<String, Object>> mesnurationRows = + mensurationByNosta.get(operationId); + + Float sampleWeight = (Float) row.get("PESP_MENS"); + if (sampleWeight == null) { + if (log.isWarnEnabled()) { + log.warn("PESP_MENS is null for operation " + operationId + "/" + speciesId); + } + sampleWeight = 0f; + } + addSubSamples(dao, + sampleDataDAO, + sampleTypeSubsample, + sampleDataTypeNumberAtLength, + sampleDataTypeWeightAtLength, + operation, + type, + species, + sampleWeight, + mesnurationRows); + + int nbSamples = operation.sizeSample(); + nbSubSample += nbSamples - 1; + } + } + + + if (log.isInfoEnabled()) { + log.info("Found " + nbSampleTotal + " 'total' sample(s)."); + log.info("Found " + nbSampleUnsorted + " 'unsorted' sample(s)."); + log.info("Found " + nbSubSample + " 'sub' sample(s)."); + log.info("Total " + (nbSampleTotal + nbSampleUnsorted + nbSubSample) + " sample(s)."); + } + } + + private void addSubSamples(TopiaDAO<Sample> dao, + SampleDataDAO sampleDataDAO, + SampleType sampleTypeSubsample, + SampleDataType sampleDataTypeNumberAtLength, + SampleDataType sampleDataTypeWeightAtLength, + Operation operation, + String type, + Species species, + Float sampleWeight, + Collection<Map<String, Object>> mesnurationRows) throws TopiaException { + + String speciesId = species.getBaracoudaCode(); + + for (Map<String, Object> row : mesnurationRows) { + String categ = (String) row.get("CATEG"); + if (!type.equals(categ)) { + + // not good category + continue; + } + String rowSpeciesId = (String) row.get("GENRE_ESP"); + if (!speciesId.equals(rowSpeciesId)) { + + // not good species + continue; + } + + // ok found a good row + Sample subsample = dao.create( + Sample.PROPERTY_SPECIES, species, + Sample.PROPERTY_SAMPLE_TYPE, sampleTypeSubsample, + Sample.PROPERTY_SAMPLE_WEIGHT, sampleWeight + ); + + Object lengthClass = row.get("TAILLE"); + + addSampleData(sampleDataDAO, + subsample, + sampleDataTypeNumberAtLength, + lengthClass, + row.get("NBIND") + ); + + addSampleData(sampleDataDAO, + subsample, + sampleDataTypeWeightAtLength, + lengthClass, + row.get("POIDSTAILLE") + ); + + + operation.addSample(subsample); + + } + } + + + protected void addSampleData(TopiaDAO<SampleData> dao, + Sample operation, + SampleDataType sampleDataType, + Object label, + Object value + ) throws TopiaException { + + if (value == null || label == null) { + return; + } + + SampleData sampleData = dao.create( + SampleData.PROPERTY_SAMPLE_DATA_TYPE, sampleDataType, + SampleData.PROPERTY_DATA_LABEL, label.toString(), + SampleData.PROPERTY_DATA_VALUE, Float.valueOf(value.toString()) + ); + operation.addSampleData(sampleData); + } + + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoader.java (rev 0) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoader.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,58 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import fr.ifremer.echobase.entities.meta.TableMeta; +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import fr.ifremer.echobase.services.models.EntityCsvModel; +import fr.ifremer.echobase.tools.FileType; + +/** + * To load {@link SpeciesCategory} entities. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class SpeciesCategoryLoader extends EntityLoader<SpeciesCategory> { + + public SpeciesCategoryLoader() { + super(SpeciesCategory.class, FileType.NEW); + } + + + @Override + protected EntityCsvModel<SpeciesCategory> createCsvImportModel(TableMeta meta) { + + EntityCsvModel<SpeciesCategory> model = EntityCsvModel.newModel( + getCsvSeparator(), + meta + ); + model.newMandatoryColumn(SpeciesCategory.PROPERTY_SPECIES); + model.newMandatoryColumn(SpeciesCategory.PROPERTY_SIZE_CATEGORY); + model.newMandatoryColumn(SpeciesCategory.PROPERTY_AGE_CATEGORY); + model.newMandatoryColumn(SpeciesCategory.PROPERTY_SEX_CATEGORY); + return model; + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoader.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AbstractLoaderIT.java =================================================================== --- trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AbstractLoaderIT.java 2011-12-15 21:53:41 UTC (rev 197) +++ trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AbstractLoaderIT.java 2011-12-16 01:46:20 UTC (rev 198) @@ -362,6 +362,9 @@ entityLoaders.add(newService(SizeCategoryLoader.class)); entityLoaders.add(newService(OperationMetadataLoader.class)); entityLoaders.add(newService(GearMetadataLoader.class)); + entityLoaders.add(newService(GearLoader.class)); + entityLoaders.add(newService(AgeCategoryLoader.class)); + entityLoaders.add(newService(SpeciesCategoryLoader.class)); entityLoaders.add(newService(VoyageLoader.class)); entityLoaders.add(newService(TransitLoader.class)); @@ -369,6 +372,7 @@ entityLoaders.add(newService(DataAcquisitionLoader.class)); entityLoaders.add(newService(DataProcessingLoader.class)); entityLoaders.add(newService(CellLoader.class)); + entityLoaders.add(newService(OperationLoader.class)); } return entityLoaders; } Added: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoaderIT.java =================================================================== --- trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoaderIT.java (rev 0) +++ trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoaderIT.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,46 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import fr.ifremer.echobase.entities.references.AgeCategory; +import org.junit.Test; + +/** + * To test the {@link AgeCategoryLoader}. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class AgeCategoryLoaderIT extends AbstractLoaderIT<AgeCategory, AgeCategoryLoader> { + + public AgeCategoryLoaderIT() { + super(AgeCategoryLoader.class, 0, 0); + } + + @Test + public void testGenerateNewEntities() throws Exception { + generateNewEntities(); + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/AgeCategoryLoaderIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/OperationLoaderIT.java =================================================================== --- trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/OperationLoaderIT.java (rev 0) +++ trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/OperationLoaderIT.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,46 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import fr.ifremer.echobase.entities.data.Operation; +import org.junit.Test; + +/** + * To test the {@link OperationLoader}. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class OperationLoaderIT extends AbstractLoaderIT<Operation, OperationLoader> { + + public OperationLoaderIT() { + super(OperationLoader.class, 0, 1374); + } + + @Test + public void testGenerateEntitiesFromBaracouda() throws Exception { + generateBaracoudaEntities(); + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/OperationLoaderIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java =================================================================== --- trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java (rev 0) +++ trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,46 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import fr.ifremer.echobase.entities.data.Sample; +import org.junit.Test; + +/** + * To test the {@link SampleLoader}. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class SampleLoaderIT extends AbstractLoaderIT<Sample, SampleLoader> { + + public SampleLoaderIT() { + super(SampleLoader.class, 0, 208199); + } + + @Test + public void testGenerateEntitiesFromBaracouda() throws Exception { + generateBaracoudaEntities(); + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoaderIT.java =================================================================== --- trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoaderIT.java (rev 0) +++ trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoaderIT.java 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1,46 @@ +/* + * #%L + * EchoBase :: Tools + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.tools.loaders; + +import fr.ifremer.echobase.entities.references.SpeciesCategory; +import org.junit.Test; + +/** + * To test the {@link SpeciesCategoryLoader}. + * + * @author tchemit <chemit@codelutin.com> + * @since 0.2 + */ +public class SpeciesCategoryLoaderIT extends AbstractLoaderIT<SpeciesCategory, SpeciesCategoryLoader> { + + public SpeciesCategoryLoaderIT() { + super(SpeciesCategoryLoader.class, 0, 0); + } + + @Test + public void testGenerateNewEntities() throws Exception { + generateNewEntities(); + } + +} \ No newline at end of file Property changes on: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SpeciesCategoryLoaderIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/test/resources/csv/new/input-AgeCategory.csv =================================================================== --- trunk/echobase-tools/src/test/resources/csv/new/input-AgeCategory.csv (rev 0) +++ trunk/echobase-tools/src/test/resources/csv/new/input-AgeCategory.csv 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1 @@ +name;meaning \ No newline at end of file Property changes on: trunk/echobase-tools/src/test/resources/csv/new/input-AgeCategory.csv ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-tools/src/test/resources/csv/new/input-SpeciesCategory.csv =================================================================== --- trunk/echobase-tools/src/test/resources/csv/new/input-SpeciesCategory.csv (rev 0) +++ trunk/echobase-tools/src/test/resources/csv/new/input-SpeciesCategory.csv 2011-12-16 01:46:20 UTC (rev 198) @@ -0,0 +1 @@ +species;sizeCategory;ageCategory;sexCategory \ No newline at end of file Property changes on: trunk/echobase-tools/src/test/resources/csv/new/input-SpeciesCategory.csv ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/src/doc/reunions/reunion-2011-12-14.txt =================================================================== --- trunk/src/doc/reunions/reunion-2011-12-14.txt 2011-12-15 21:53:41 UTC (rev 197) +++ trunk/src/doc/reunions/reunion-2011-12-14.txt 2011-12-16 01:46:20 UTC (rev 198) @@ -49,6 +49,8 @@ - Remplissage du référentiel GearMetadata - Remplissage du référentiel Gear +- Import des opérations (depuis table STATIONS) + Action ------