This is an automated email from the git hooks/post-receive script. New commit to branch feature/importBigfin in repository tutti. See http://git.codelutin.com/tutti.git commit 24822a38625bbccb83298b406e5e9b145d8788ab Author: Kevin Morin <morin@codelutin.com> Date: Tue Sep 23 14:24:39 2014 +0200 fixes #5411 [CAPTURE] Import BIGFIN --- .../tutti/service/bigfin/BigfinDataRowModel.java | 7 +- .../tutti/service/bigfin/BigfinImportResult.java | 30 ++--- .../tutti/service/bigfin/BigfinImportService.java | 95 +++++++------- .../fr/ifremer/tutti/service/bigfin/Signs.java | 19 +-- .../resources/i18n/tutti-service_en_GB.properties | 5 + .../resources/i18n/tutti-service_fr_FR.properties | 9 +- .../service/bigfin/BigfinImportServiceTest.java | 142 ++++++++++----------- .../test/resources/bigfin/importbigfin-errors.csv | 4 + .../test/resources/bigfin/importbigfin-invalid.csv | 31 ----- .../test/resources/bigfin/importbigfin-valid.csv | 53 ++++---- .../resources/bigfin/importbigfin-warnings.csv | 5 + tutti-service/src/test/resources/log4j.properties | 1 + .../tutti/ui/swing/action/ImportBigfinAction.java | 6 +- 13 files changed, 195 insertions(+), 212 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java index dd94d7c..a4500c6 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinDataRowModel.java @@ -5,6 +5,7 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId; import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId2; import fr.ifremer.tutti.persistence.entities.TuttiEntities; import fr.ifremer.tutti.persistence.entities.protocol.SpeciesProtocol; @@ -86,7 +87,7 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi new ValueParser<Signs>() { @Override public Signs parse(String value) throws ParseException { - Signs result = Signs.getSign(value.toUpperCase(), PmfmId2.SIZE_CATEGORY.getValue()); + Signs result = Signs.getSign(value.toUpperCase(), PmfmId.SIZE_CATEGORY.getValue()); return result; } }); @@ -96,7 +97,7 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi new ValueParser<Signs>() { @Override public Signs parse(String value) throws ParseException { - Signs result = Signs.getSign(value.toUpperCase(), PmfmId2.SEX.getValue()); + Signs result = Signs.getSign(value.toUpperCase(), PmfmId.SEX.getValue()); if (result == null) { throw new ParseException("Could not parse Sign value: " + value, 0); } @@ -108,7 +109,7 @@ public class BigfinDataRowModel extends TuttiCsvUtil.AbstractTuttiImportModel<Bi @Override public Signs parse(String value) throws ParseException { // On importe dans le Vrac par défaut, sauf si il y a le texte HV ou hv dans le champ text - Signs result = Signs.getSign(value.toUpperCase(), PmfmId2.SORTED_UNSORTED.getValue()); + Signs result = Signs.getSign(value.toUpperCase(), PmfmId.SORTED_UNSORTED.getValue()); if (result == null) { result = Signs.VRAC; } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java index 407f10c..7f4d1da 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java @@ -38,16 +38,14 @@ public class BigfinImportResult { protected final File importFile; - protected final List<String> fatalErrors = new ArrayList<>(); - protected final List<String> errors = new ArrayList<>(); + protected final List<String> warnings = new ArrayList<>(); + protected int nbFrequenciesImported; protected int nbFrequenciesDeleted; - protected final List<Species> speciesNotInProtocol = new ArrayList<>(); - public BigfinImportResult(File importFile) { this.importFile = importFile; } @@ -64,35 +62,31 @@ public class BigfinImportResult { return nbFrequenciesDeleted; } - public List<String> getFatalErrors() { - return fatalErrors; - } - public List<String> getErrors() { return errors; } - void incrementNbFrequenciesImported(int nb) { - this.nbFrequenciesImported += nb; + public List<String> getWarnings() { + return warnings; } - void incrementNbFrequenciesDeleted(int nb) { - this.nbFrequenciesDeleted += nb; + public void incrementNbFrequenciesImported(int nb) { + this.nbFrequenciesImported += nb; } - void addFatalError(String error) { - fatalErrors.add(error); + public void incrementNbFrequenciesDeleted(int nb) { + this.nbFrequenciesDeleted += nb; } - void addError(String error) { + public void addError(String error) { errors.add(error); } - void addSpeciesNotInProtocol(Species species) { - speciesNotInProtocol.add(species); + public void addWarning(String warning) { + warnings.add(warning); } public boolean isDone() { - return fatalErrors.isEmpty(); + return errors.isEmpty(); } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java index 1f40312..f06b104 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java @@ -9,9 +9,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.common.io.Files; -import fr.ifremer.adagio.core.dao.referential.pmfm.ObjectTypeCode2; +import fr.ifremer.adagio.core.dao.referential.ObjectTypeCode; import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId; -import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValue; import fr.ifremer.tutti.persistence.entities.data.Attachment; import fr.ifremer.tutti.persistence.entities.data.Attachments; import fr.ifremer.tutti.persistence.entities.data.BatchContainer; @@ -118,31 +117,32 @@ public class BigfinImportService extends AbstractTuttiService { List<Species> allSpeciesWithSurveyCode = persistenceService.getReferentSpeciesWithSurveyCode(allReferentSpecies); Map<String, Species> speciesBySurveyCode = Maps.newTreeMap(); - for (Species species : allSpeciesWithSurveyCode) { + for (Species species : allReferentSpecies) { String surveyCode = species.getSurveyCode(); if (StringUtils.isNotBlank(surveyCode)) { speciesBySurveyCode.put(surveyCode, species); - } else { + } else if (species.getRefTaxCode() != null) { speciesBySurveyCode.put(species.getRefTaxCode(), species); } } - speciesProtocolBySurveyCode = Maps.newTreeMap(); + speciesProtocolBySurveyCode = Maps.newTreeMap(); for (SpeciesProtocol speciesProtocol : protocol.getSpecies()) { - if (speciesProtocol.getSpeciesSurveyCode() != null) { + if (StringUtils.isNotBlank(speciesProtocol.getSpeciesSurveyCode())) { speciesProtocolBySurveyCode.put(speciesProtocol.getSpeciesSurveyCode(), speciesProtocol); } else { speciesProtocolBySurveyCode.put(speciesProtocol.getSpeciesReferenceTaxonId().toString(), speciesProtocol); } - } BatchContainer<SpeciesBatch> rootSpeciesBatch = persistenceService.getRootSpeciesBatch(operation.getId(), false); // set of not found species already added in the errors Set<Species> speciesNotRecognized = new HashSet<>(); + //set of species not in the protocol + Set<Species> speciesNotInProtocol = new HashSet<>(); // set of species without lengthstep alreay added in the errors Set<Species> speciesInProtocolButWithoutLengthStepPmfmId = new HashSet<>(); @@ -170,47 +170,54 @@ public class BigfinImportService extends AbstractTuttiService { if (bean.getSzClass() == null) { String error = t("tutti.service.bigfinImport.error.szClass.unknwon", bean.getRecordId()); - if (log.isWarnEnabled()) { - log.warn(error); + if (log.isErrorEnabled()) { + log.error(error); } - result.addFatalError(error); + result.addError(error); } if (species == null || species.getId() == null) { // bloquer tout si un "species" ne match pas le référentiel de Tutti : lister dans ce cas les codes non reconnus if (speciesNotRecognized.add(species)) { String error = t("tutti.service.bigfinImport.error.species.not.found", species.getExternalCode()); - if (log.isWarnEnabled()) { - log.warn(error); + if (log.isErrorEnabled()) { + log.error(error); } - result.addFatalError(error); + result.addError(error); } } else { String code = species.getSurveyCode(); - if (code == null) { + if (StringUtils.isBlank(code)) { code = species.getReferenceTaxonId().toString(); } SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(code); + String speciesLabel = species.getSurveyCode(); + if (StringUtils.isBlank(speciesLabel)) { + speciesLabel = species.getRefTaxCode(); + } + // On n'importe pas les espèces non présentes dans le protocole et // on liste les espèces/catégorisées non importées pour aider l'utilisateur // à identifier le problème et on fait l'import des autres if (speciesProtocol == null) { - result.addSpeciesNotInProtocol(species); - - } else if (speciesProtocol.getLengthStepPmfmId() == null - && speciesInProtocolButWithoutLengthStepPmfmId.add(species)) { -// bloquer toute espèce reconnue du protocole mais qui n'a pas de méthode de mesure - String speciesLabel = species.getSurveyCode(); - if (StringUtils.isBlank(speciesLabel)) { - speciesLabel = species.getRefTaxCode(); + if (speciesNotInProtocol.add(species)) { + String error = t("tutti.service.bigfinImport.warning.species.notInProtocol", speciesLabel); + if (log.isWarnEnabled()) { + log.warn(error); + } + result.addWarning(error); } - String error = t("tutti.service.bigfinImport.error.species.without.lengthstep", speciesLabel); - if (log.isWarnEnabled()) { - log.warn(error); - } - result.addFatalError(error); + } else if (speciesProtocol.getLengthStepPmfmId() == null) { + if (speciesInProtocolButWithoutLengthStepPmfmId.add(species)) { +// bloquer toute espèce reconnue du protocole mais qui n'a pas de méthode de mesure + String error = t("tutti.service.bigfinImport.error.species.without.lengthstep", speciesLabel); + if (log.isErrorEnabled()) { + log.error(error); + } + result.addError(error); + } } else { rows.add(bean); } @@ -254,7 +261,7 @@ public class BigfinImportService extends AbstractTuttiService { // get the speciesprotocol and its lengthstep pmfm String code = species.getSurveyCode(); - if (code == null) { + if (StringUtils.isBlank(code)) { code = species.getReferenceTaxonId().toString(); } SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(code); @@ -364,10 +371,10 @@ public class BigfinImportService extends AbstractTuttiService { if (category.getCategoryValueGetter() == null) { // if the batch is not the last one, error, we cannot add the frequencies to a more categorized batch if (CollectionUtils.isNotEmpty(batch.getChildBatchs())) { - commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.tooCategorized", - commonParameter.getSpeciesLabel(), - sizeCaracteristic.getParameterName(), - genderCaracteristic.getParameterName())); + commonParameter.getResult().addWarning(t("tutti.service.bigfinImport.warning.species.tooCategorized", + commonParameter.getSpeciesLabel(), + sizeCaracteristic.getParameterName(), + genderCaracteristic.getParameterName())); } else { // create the frequencies @@ -382,8 +389,8 @@ public class BigfinImportService extends AbstractTuttiService { } else if (batchHasFrequencies) { // if the batch is supposed to be categorized again, but already has frequencies CaracteristicQualitativeValue qualitativeValue = signsToCaracteristicValue.get(caracteristic); - commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.batch.frequenciesOnHigherLevel", - commonParameter.getSpeciesLabel(), qualitativeValue.getName())); + commonParameter.getResult().addWarning(t("tutti.service.bigfinImport.warning.species.batch.frequenciesOnHigherLevel", + commonParameter.getSpeciesLabel(), qualitativeValue.getName())); } else { List<SpeciesBatch> batchChildren = batch.getChildBatchs(); @@ -411,19 +418,19 @@ public class BigfinImportService extends AbstractTuttiService { // check that this time, it is the right category. We can only skip one nextCategory = commonParameter.getCategories().get(depth); if (!nextCategory.getPmfmId().equals(categoryId)) { - commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.categoriesSkipped", - commonParameter.getSpeciesLabel(), - sizeCaracteristic.getParameterName(), - genderCaracteristic.getParameterName() - )); + commonParameter.getResult().addWarning(t("tutti.service.bigfinImport.warning.species.categoriesSkipped", + commonParameter.getSpeciesLabel(), + sizeCaracteristic.getParameterName(), + genderCaracteristic.getParameterName() + )); continue; } } else { - commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.categorySkipped", - commonParameter.getSpeciesLabel(), - persistenceService.getCaracteristic(nextCategory.getPmfmId()).getParameterName() - )); + commonParameter.getResult().addWarning(t("tutti.service.bigfinImport.warning.species.categorySkipped", + commonParameter.getSpeciesLabel(), + persistenceService.getCaracteristic(nextCategory.getPmfmId()).getParameterName() + )); continue; } } @@ -513,7 +520,7 @@ public class BigfinImportService extends AbstractTuttiService { protected void addFileAsAttachment(File f, CatchBatch catchBatch) { Attachment attachment = Attachments.newAttachment(); - attachment.setObjectType(ObjectTypeCode2.CATCH_BATCH); + attachment.setObjectType(ObjectTypeCode.CATCH_BATCH); attachment.setObjectId(Integer.valueOf(catchBatch.getId())); attachment.setName(f.getName()); String date = DateFormat.getDateTimeInstance().format(context.currentDate()); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java index 208325e..0e0cec8 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java @@ -22,6 +22,7 @@ package fr.ifremer.tutti.service.bigfin; * #L% */ +import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId; import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId2; import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValueId; import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValueId2; @@ -43,7 +44,7 @@ public enum Signs { NOT_SEXED("NOT SET") { @Override public Integer getCategory() { - return PmfmId2.SEX.getValue(); + return PmfmId.SEX.getValue(); } @Override @@ -59,7 +60,7 @@ public enum Signs { MALE("M") { @Override public Integer getCategory() { - return PmfmId2.SEX.getValue(); + return PmfmId.SEX.getValue(); } @Override @@ -75,7 +76,7 @@ public enum Signs { FEMALE("F") { @Override public Integer getCategory() { - return PmfmId2.SEX.getValue(); + return PmfmId.SEX.getValue(); } @Override @@ -93,12 +94,12 @@ public enum Signs { NOT_SIZED("0") { @Override public Integer getCategory() { - return PmfmId2.SIZE_CATEGORY.getValue(); + return PmfmId.SIZE_CATEGORY.getValue(); } @Override public Integer getQualitativeValueId() { - return QualitativeValueId2.NOT_SIZED.getValue(); + return QualitativeValueId.UNSORTED.getValue(); } @Override @@ -110,7 +111,7 @@ public enum Signs { SMALL("1") { @Override public Integer getCategory() { - return PmfmId2.SIZE_CATEGORY.getValue(); + return PmfmId.SIZE_CATEGORY.getValue(); } @Override @@ -126,7 +127,7 @@ public enum Signs { BIG("2") { @Override public Integer getCategory() { - return PmfmId2.SIZE_CATEGORY.getValue(); + return PmfmId.SIZE_CATEGORY.getValue(); } @Override @@ -144,7 +145,7 @@ public enum Signs { VRAC("") { @Override public Integer getCategory() { - return PmfmId2.SORTED_UNSORTED.getValue(); + return PmfmId.SORTED_UNSORTED.getValue(); } @Override @@ -160,7 +161,7 @@ public enum Signs { HORS_VRAC("HV") { @Override public Integer getCategory() { - return PmfmId2.SORTED_UNSORTED.getValue(); + return PmfmId.SORTED_UNSORTED.getValue(); } @Override diff --git a/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties b/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties index 7c70c0e..d686b05 100644 --- a/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties +++ b/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties @@ -55,6 +55,11 @@ tutti.service.bigfinImport.error.species.tooCategorized= tutti.service.bigfinImport.error.species.without.lengthstep= tutti.service.bigfinImport.error.species.wrongNextCategory= tutti.service.bigfinImport.error.szClass.unknwon= +tutti.service.bigfinImport.warning.species.batch.frequenciesOnHigherLevel= +tutti.service.bigfinImport.warning.species.categoriesSkipped= +tutti.service.bigfinImport.warning.species.categorySkipped= +tutti.service.bigfinImport.warning.species.notInProtocol= +tutti.service.bigfinImport.warning.species.tooCategorized= tutti.service.bigfinimport.error.no.protocol= tutti.service.compressZipFile.error= tutti.service.context.serviceInstanciation.error= diff --git a/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties b/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties index 5446ee6..14cec28 100644 --- a/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties +++ b/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties @@ -44,13 +44,14 @@ tutti.report.step.export.fishingOperation=Exporter le trait sélectionné tutti.report.step.generateReport=Générer le rapport tutti.report.step.load.fishingOperation=Charger le trait sélectionné tutti.service.bigfin.import.attachment.comment=Import Bigfin du %s -tutti.service.bigfinImport.error.species.batch.frequenciesOnHigherLevel=Le lot de '<strong>%1s / %2s</strong>' a déjà des mensurations -tutti.service.bigfinImport.error.species.categoriesSkipped=L'espèce '<strong>%1s</strong>' a été catégorisée sans les catégories '<strong>%2s</strong>' et '<strong>%3s</strong>' -tutti.service.bigfinImport.error.species.categorySkipped=L'espèce '<strong>%1s</strong>' a été catégorisée sans la catégorie '<strong>%2s</strong>' tutti.service.bigfinImport.error.species.not.found=L'espèce '<strong>%s</strong>' est inconnue -tutti.service.bigfinImport.error.species.tooCategorized=L'espèce '<strong>%1s</strong>' est trop catégorisée (pas limitée à '<strong>%2s</strong>' et '<strong>%3s</strong>') tutti.service.bigfinImport.error.species.without.lengthstep=L'espèce '<strong>%s</strong>' n'a pas de classe de taille associée dans le protocole. tutti.service.bigfinImport.error.szClass.unknwon=Ligne <i>%s</i>, code inconnu (doit être 0, 1 ou 2) +tutti.service.bigfinImport.warning.species.batch.frequenciesOnHigherLevel=Le lot de '<strong>%1s / %2s</strong>' a déjà des mensurations +tutti.service.bigfinImport.warning.species.categoriesSkipped=L'espèce '<strong>%1s</strong>' a été catégorisée sans les catégories '<strong>%2s</strong>' et '<strong>%3s</strong>' +tutti.service.bigfinImport.warning.species.categorySkipped=L'espèce '<strong>%1s</strong>' a été catégorisée sans la catégorie '<strong>%2s</strong>' +tutti.service.bigfinImport.warning.species.notInProtocol=L'espèce '<strong>%1s</strong>' n'est pas présente dans le protocole +tutti.service.bigfinImport.warning.species.tooCategorized=L'espèce '<strong>%1s</strong>' est trop catégorisée (pas limitée à '<strong>%2s</strong>' et '<strong>%3s</strong>') tutti.service.bigfinimport.error.no.protocol=Impossible de faire un import Bigfin sans protocol. tutti.service.compressZipFile.error=Erreur lors de la compression du dossier %1s dans le fichier %2s tutti.service.context.serviceInstanciation.error=Erreur lors de l'instanciation du service %s diff --git a/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java b/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java index 2539627..85c48df 100644 --- a/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java +++ b/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java @@ -1,10 +1,8 @@ package fr.ifremer.tutti.service.bigfin; import com.google.common.base.Predicate; -import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId2; +import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId; import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValueId; -import fr.ifremer.tutti.TuttiConfigurationOption; -import fr.ifremer.tutti.persistence.ProgressionModel; import fr.ifremer.tutti.persistence.entities.data.BatchContainer; import fr.ifremer.tutti.persistence.entities.data.CatchBatch; import fr.ifremer.tutti.persistence.entities.data.FishingOperation; @@ -21,14 +19,11 @@ import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.nuiton.jaxx.application.ApplicationBusinessException; -import sun.reflect.annotation.ExceptionProxy; import java.io.File; import java.io.IOException; import java.util.List; -import static org.nuiton.i18n.I18n.t; - /** * @author Kevin Morin (Code Lutin) * @since 3.8 @@ -36,16 +31,16 @@ import static org.nuiton.i18n.I18n.t; public class BigfinImportServiceTest { @ClassRule - public static final ServiceDbResource dbResource = ServiceDbResource.writeDb("dbMedits"); + public static final ServiceDbResource dbResource = ServiceDbResource.writeDb("dbImportBigfin"); /** Logger. */ private static final Log log = LogFactory.getLog(BigfinImportServiceTest.class); - public static final String PROGRAM_ID = "CAM-MEDITS"; + public static final String PROGRAM_ID = "CAM-TEST_"; - public static final String CRUISE_ID = "100001"; + public static final String CRUISE_ID = "0"; - public static final String OPERATION_1_ID = "100106"; + public static final String OPERATION_1_ID = "0"; protected BigfinImportService service; @@ -53,8 +48,6 @@ public class BigfinImportServiceTest { protected ServiceDbResource.DataContext dataContext; - protected ProgressionModel progressionModel; - protected File dataDirectory; protected Predicate<SpeciesAbleBatch> vracPredicate; @@ -66,21 +59,16 @@ public class BigfinImportServiceTest { TuttiServiceContext serviceContext = dbResource.getServiceContext(); - File protocol = dbResource.copyClassPathResource("bigfin/protocol.tuttiProtocol", "protocol.tuttiProtocol"); - dbResource.getConfig().getApplicationConfig().setOption(TuttiConfigurationOption.DB_PROTOCOL_DIRECTORY.getKey(), protocol.getParentFile().getAbsolutePath()); - serviceContext.getDataContext().setProtocolId("protocol"); + dbResource.loadInternalProtocolFile(); dbResource.openDataContext(); persistenceService = serviceContext.getService(PersistenceService.class); service = serviceContext.getService(BigfinImportService.class); - progressionModel = new ProgressionModel(); - progressionModel.setTotal(9); - dataContext = dbResource.loadContext(PROGRAM_ID, CRUISE_ID, 1, OPERATION_1_ID); - vracPredicate = SpeciesAbleBatchs.newSpeciesAbleBatchCategoryPredicate(PmfmId2.SORTED_UNSORTED.getValue(), QualitativeValueId.VRAC.getValue()); + vracPredicate = SpeciesAbleBatchs.newSpeciesAbleBatchCategoryPredicate(PmfmId.SORTED_UNSORTED.getValue(), QualitativeValueId.VRAC.getValue()); } @Test @@ -88,102 +76,73 @@ public class BigfinImportServiceTest { File importFile = dbResource.copyClassPathResource("bigfin/importbigfin-valid.csv", "importbigfin-valid.csv"); - FishingOperation operation = dataContext.operations.get(1); + FishingOperation operation = dataContext.operations.get(0); CatchBatch catchBatch = persistenceService.getCatchBatchFromFishingOperation(operation.getId()); catchBatch.setFishingOperation(operation); - BatchContainer<SpeciesBatch> rootSpeciesBatch = persistenceService.getRootSpeciesBatch(operation.getId(), false); - for (SpeciesBatch speciesBatch : rootSpeciesBatch.getChildren()) { - persistenceService.deleteSpeciesBatch(speciesBatch.getId()); - } - BigfinImportResult importResult = service.importFile(importFile, operation, catchBatch); int nbFrequenciesAdded = importResult.getNbFrequenciesImported(); - List<String> fatals = importResult.getFatalErrors(); List<String> errors = importResult.getErrors(); + List<String> warnings = importResult.getWarnings(); if (log.isInfoEnabled()) { log.info("Frequencies Imported: " + nbFrequenciesAdded); - log.info("Fatals: " + fatals.size()); log.info("Errors: " + errors.size()); + log.info("Warnings: " + warnings.size()); } - int nbNewFrequencies = 3; + int nbNewFrequencies = 32; Assert.assertEquals(nbNewFrequencies, nbFrequenciesAdded); - Assert.assertEquals(0, fatals.size()); Assert.assertEquals(0, errors.size()); + Assert.assertEquals(0, warnings.size()); // no batch imported BatchContainer<SpeciesBatch> rootSpeciesBatchAfter = persistenceService.getRootSpeciesBatch(operation.getId(), false); -// int totalSortedBatchs = 0; -// int totalUnsortedBatchs = 0; -// for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { -// -// boolean sorted = vracPredicate.apply(speciesBatch); -// -// if (sorted) { -// totalSortedBatchs++; -// } else { -// totalUnsortedBatchs++; -// } -// } -// -// Assert.assertEquals(nbNewSortedBatchs, totalSortedBatchs); -// Assert.assertEquals(nbNewUnsortedBatchs, totalUnsortedBatchs); + int totalSortedBatchs = 0; + int totalUnsortedBatchs = 0; + for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { + + boolean sorted = vracPredicate.apply(speciesBatch); + + if (sorted) { + totalSortedBatchs++; + } else { + totalUnsortedBatchs++; + } + } + + Assert.assertEquals(4, totalSortedBatchs); + Assert.assertEquals(3, totalUnsortedBatchs); } @Test - public void importInvalid() throws IOException { + public void importWithErrors() throws IOException { - File importFile = dbResource.copyClassPathResource("bigfin/importbigfin-invalid.csv", "importbigfin-invalid.csv"); + File importFile = dbResource.copyClassPathResource("bigfin/importbigfin-errors.csv", "importbigfin-invalid-errors.csv"); - FishingOperation operation = dataContext.operations.get(1); + FishingOperation operation = dataContext.operations.get(0); CatchBatch catchBatch = persistenceService.getCatchBatchFromFishingOperation(operation.getId()); catchBatch.setFishingOperation(operation); - BatchContainer<SpeciesBatch> rootSpeciesBatch = persistenceService.getRootSpeciesBatch(operation.getId(), false); - for (SpeciesBatch speciesBatch : rootSpeciesBatch.getChildren()) { - persistenceService.deleteSpeciesBatch(speciesBatch.getId()); - } - BigfinImportResult importResult = service.importFile(importFile, operation, catchBatch); int nbFrequenciesAdded = importResult.getNbFrequenciesImported(); - List<String> fatals = importResult.getFatalErrors(); List<String> errors = importResult.getErrors(); + List<String> warnings = importResult.getWarnings(); + if (log.isInfoEnabled()) { log.info("Frequencies Imported: " + nbFrequenciesAdded); - log.info("Fatals: " + fatals.size()); log.info("Errors: " + errors.size()); + log.info("Warnings: " + warnings.size()); } int nbNewFrequencies = 0; - int nbFatals = 3; - int nbErrors = 0; Assert.assertEquals(nbNewFrequencies, nbFrequenciesAdded); - Assert.assertEquals(nbFatals, fatals.size()); - Assert.assertEquals(nbErrors, errors.size()); - - // no batch imported - BatchContainer<SpeciesBatch> rootSpeciesBatchAfter = persistenceService.getRootSpeciesBatch(operation.getId(), false); - -// int totalSortedBatchs = 0; -// for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { -// -// boolean sorted = vracPredicate.apply(speciesBatch); -// -// if (sorted) { -// totalSortedBatchs++; -// } else { -// totalUnsortedBatchs++; -// } -// } - -// Assert.assertEquals(nbFrequenciesAdded, totalSortedBatchs); - + Assert.assertEquals(2, errors.size()); + Assert.assertEquals(0, warnings.size()); TuttiServiceContext serviceContext = dbResource.getServiceContext(); serviceContext.getDataContext().resetProtocol(); @@ -192,8 +151,39 @@ public class BigfinImportServiceTest { try { service.importFile(importFile, operation, catchBatch); } catch (ApplicationBusinessException e) { + if (log.isErrorEnabled()) { + log.error("Error during import: " + e.getMessage()); + } failIfNoProtocol = true; } Assert.assertTrue(failIfNoProtocol); } + + @Test + public void importWithWarnings() throws IOException { + + File importFile = dbResource.copyClassPathResource("bigfin/importbigfin-warnings.csv", "importbigfin-invalid-warnings.csv"); + + FishingOperation operation = dataContext.operations.get(0); + CatchBatch catchBatch = persistenceService.getCatchBatchFromFishingOperation(operation.getId()); + catchBatch.setFishingOperation(operation); + + BigfinImportResult importResult = service.importFile(importFile, operation, catchBatch); + + int nbFrequenciesAdded = importResult.getNbFrequenciesImported(); + List<String> errors = importResult.getErrors(); + List<String> warnings = importResult.getWarnings(); + + if (log.isInfoEnabled()) { + log.info("Frequencies Imported: " + nbFrequenciesAdded); + log.info("Errors: " + errors.size()); + log.info("Warnings: " + warnings.size()); + } + + int nbNewFrequencies = 1; + Assert.assertEquals(nbNewFrequencies, nbFrequenciesAdded); + Assert.assertEquals(0, errors.size()); + Assert.assertEquals(3, warnings.size()); + + } } diff --git a/tutti-service/src/test/resources/bigfin/importbigfin-errors.csv b/tutti-service/src/test/resources/bigfin/importbigfin-errors.csv new file mode 100644 index 0000000..378a0dc --- /dev/null +++ b/tutti-service/src/test/resources/bigfin/importbigfin-errors.csv @@ -0,0 +1,4 @@ +Record_id;length(mm);weight(g);species;cruise;station;pan;Dt;sz class;gender;text +0;98;250;NOLSESP;;1;VB;05/31/14 20:20;1;Not set;libre +1;185;215;ESP1;;1;VB;05/31/14 20:20;1;Not set; +2;156;321;ESP1;;1;VB;05/31/14 20:20;6;Not set; diff --git a/tutti-service/src/test/resources/bigfin/importbigfin-invalid.csv b/tutti-service/src/test/resources/bigfin/importbigfin-invalid.csv deleted file mode 100644 index 4d5dabe..0000000 --- a/tutti-service/src/test/resources/bigfin/importbigfin-invalid.csv +++ /dev/null @@ -1,31 +0,0 @@ -Record_id;length(mm);weight(g);species;cruise;station;pan;Dt;sz class;gender;text -0;98;250;ARISFOL;THALASSA;1;VB;05/31/14 20:20;42;Not set;libre -1;123;360;ARISFOL;THALASSA;1;VB;05/31/14 20:20;1;Not set; -2;185;215;ARISFOL;THALASSA;1;VB;05/31/14 20:20;1;Not set; -3;101;145;ARISFOL;THALASSA;1;VB;05/31/14 20:20;1;Not set; -5;90;256;ARISFOL;THALASSA;1;VB;05/31/14 20:20;2;Not set; -6;131;258;ARISFOL;THALASSA;1;VB;05/31/14 20:20;2;Not set; -7;168;365;ARISFOL;THALASSA;1;VB;05/31/14 20:20;2;Not set; -8;165;324;ARISFOL;THALASSA;1;VB;05/31/14 20:20;2;Not set; -9;136;124;ARISFOL;THALASSA;1;VB;05/31/14 20:20;2;Not set; -10;145;241;ARISFOL;THALASSA;1;VB;05/31/14 20:21;2;Not set; -11;125;256;ARISFOL;THALASSA;1;VB;05/31/14 20:22;2;Not set; -12;650;0;ARITANT;THALASSA;1;VB;05/31/14 20:23;0;M; -13;654;0;ARITANT;THALASSA;1;VB;05/31/14 20:24;0;M; -14;521;0;ARITANT;THALASSA;1;VB;05/31/14 20:25;0;F; -15;587;0;ARITANT;THALASSA;1;VB;05/31/14 20:26;0;F; -16;456;0;ARITANT;THALASSA;1;VB;05/31/14 20:27;0;M; -17;562;0;ARITANT;THALASSA;1;VB;05/31/14 20:28;0;F; -18;562;0;ARITANT;THALASSA;1;VB;05/31/14 20:29;0;F; -19;125;1025;TRACTRU;THALASSA;1;VB;05/31/14 20:30;0;Not set; -20;145;0;TRACTRU;THALASSA;1;VB;05/31/14 20:31;0;Not set; -21;102;0;TRACTRU;THALASSA;1;VB;05/31/14 20:32;0;Not set; -22;96;0;TRACTRU;THALASSA;1;VB;05/31/14 20:33;0;Not set; -23;84;0;TRACTRU;THALASSA;1;VB;05/31/14 20:34;0;Not set; -24;84;0;TRACTRU;THALASSA;1;VB;05/31/14 20:35;0;Not set; -25;132;0;TRACTRU;THALASSA;1;VB;05/31/14 20:36;0;Not set; -26;105;0;TRACTRU;THALASSA;1;VB;05/31/14 20:37;0;Not set; -27;80;0;TRACTRU;THALASSA;1;VB;05/31/14 20:38;0;Not set; -28;96;0;TRACTRU;THALASSA;1;VB;05/31/14 20:39;0;Not set; -29;87;0;PAPELON;THALASSA;1;VB;05/31/14 20:40;0;Not set; -30;785;1240;DICE-LAB;THALASSA;1;VB;05/31/14 20:41;2;M; diff --git a/tutti-service/src/test/resources/bigfin/importbigfin-valid.csv b/tutti-service/src/test/resources/bigfin/importbigfin-valid.csv index 72e68c9..bf60730 100644 --- a/tutti-service/src/test/resources/bigfin/importbigfin-valid.csv +++ b/tutti-service/src/test/resources/bigfin/importbigfin-valid.csv @@ -1,31 +1,36 @@ Record_id;length(mm);weight(g);species;cruise;station;pan;Dt;sz class;gender;text -0;98;250;RAJAMON;;1;VB;05/31/14 20:20;1;Not set;libre -1;123;360;RAJAMON;;1;VB;05/31/14 20:20;1;Not set; -2;185;215;RAJAMON;;1;VB;05/31/14 20:20;1;Not set; -3;101;145;RAJAMON;;1;VB;05/31/14 20:20;1;Not set; -5;90;256;RAJAMON;;1;VB;05/31/14 20:20;2;Not set; -6;131;258;RAJAMON;;1;VB;05/31/14 20:20;2;Not set; -7;168;365;RAJAMON;;1;VB;05/31/14 20:20;2;Not set; -8;165;324;RAJAMON;;1;VB;05/31/14 20:20;2;Not set; -9;136;124;RAJAMON;;1;VB;05/31/14 20:20;2;Not set; -10;145;241;RAJAMON;;1;VB;05/31/14 20:21;2;Not set; -11;125;256;RAJAMON;;1;VB;05/31/14 20:22;2;Not set; -12;650;0;DICE-LAB;;1;VB;05/31/14 20:23;0;M; +0;98;250;ESP1;;1;VB;05/31/14 20:20;1;Not set;libre +1;123;360;ESP1;;1;VB;05/31/14 20:20;1;Not set; +2;185;215;ESP1;;1;VB;05/31/14 20:20;1;Not set; +3;101;145;ESP1;;1;VB;05/31/14 20:20;1;M; +4;90;256;ESP1;;1;VB;05/31/14 20:20;2;M; +5;90;256;ESP1;;1;VB;05/31/14 20:20;2;M; +6;131;258;ESP1;;1;VB;05/31/14 20:20;2;Not set; +7;168;365;ESP1;;1;VB;05/31/14 20:20;2;Not set; +8;165;324;ESP1;;1;VB;05/31/14 20:20;2;M; +9;136;124;ESP1;;1;VB;05/31/14 20:20;2;M; +10;145;241;ESP2;;1;VB;05/31/14 20:21;2;M;hv +11;125;256;ESP2;;1;VB;05/31/14 20:22;2;M;hV +12;650;0;DICELAB;;1;VB;05/31/14 20:23;0;M; 13;654;0;DICELAB;;1;VB;05/31/14 20:24;0;M; 14;521;0;DICELAB;;1;VB;05/31/14 20:25;0;F; 15;587;0;DICELAB;;1;VB;05/31/14 20:26;0;F; 16;456;0;DICELAB;;1;VB;05/31/14 20:27;0;M; 17;562;0;DICELAB;;1;VB;05/31/14 20:28;0;F; 18;562;0;DICELAB;;1;VB;05/31/14 20:29;0;F; -19;125;1025;TRACTRU;;;VB;05/31/14 20:30;0;Not set; -20;145;0;TRACTRU;;;VB;05/31/14 20:31;0;Not set; -21;102;0;TRACTRU;;;VB;05/31/14 20:32;0;Not set; -22;96;0;TRACTRU;;;VB;05/31/14 20:33;0;Not set; -23;84;0;TRACTRU;;;VB;05/31/14 20:34;0;Not set; -24;84;0;TRACTRU;;;VB;05/31/14 20:35;0;Not set; -25;132;0;TRACTRU;;;VB;05/31/14 20:36;0;Not set; -26;105;0;TRACTRU;;;VB;05/31/14 20:37;0;Not set; -27;80;0;TRACTRU;;;VB;05/31/14 20:38;0;Not set; -28;96;0;TRACTRU;;;VB;05/31/14 20:39;0;Not set; -29;87;0;TRACTRU;;;VB;05/31/14 20:40;0;Not set; -30;785;1240;DICE-LAB;;1;VB;05/31/14 20:41;2;M;HV +19;125;0;ESP2;;1;VB;05/31/14 20:30;0;M; +20;145;0;ESP2;;1;VB;05/31/14 20:31;0;M; +21;102;0;ESP2;;1;VB;05/31/14 20:32;0;M; +22;96;125;ESP2;;1;VB;05/31/14 20:33;0;F; +23;84;412;ESP2;;1;VB;05/31/14 20:34;0;F; +24;84;362;ESP2;;1;VB;05/31/14 20:35;0;F; +25;132;865;ESP2;;1;VB;05/31/14 20:36;0;F; +26;105;254;ESP2;;1;VB;05/31/14 20:37;0;F; +27;80;0;ESP3;;1;VB;05/31/14 20:38;0;Not set; +28;96;0;ESP3;;1;VB;05/31/14 20:39;0;Not set; +29;87;0;ESP3;;1;VB;05/31/14 20:40;0;Not set; +30;785;1240;DICELAB;1;1;VB;05/31/14 20:41;2;M;Hv +31;780;1220;DICELAB;1;1;VB;05/31/14 20:41;2;M;HV +32;80;0;ESP3;;2;VB;05/31/14 20:38;0;Not set; +33;96;0;ESP3;;1;VB;05/31/13 20:39;0;Not set; +34;87;0;ESP3;;1;VB;05/31/15 20:40;0;Not set; diff --git a/tutti-service/src/test/resources/bigfin/importbigfin-warnings.csv b/tutti-service/src/test/resources/bigfin/importbigfin-warnings.csv new file mode 100644 index 0000000..48dc309 --- /dev/null +++ b/tutti-service/src/test/resources/bigfin/importbigfin-warnings.csv @@ -0,0 +1,5 @@ +Record_id;length(mm);weight(g);species;cruise;station;pan;Dt;sz class;gender;text +0;98;250;ABAL;;1;VB;05/31/14 20:20;1;Not set;libre +1;185;215;ESP1;;1;VB;05/31/14 20:20;1;F; +2;156;321;ESP1;;1;VB;05/31/14 20:20;1;Not set; +3;156;321;ESP1;;1;VB;05/31/14 20:20;1;F;hv \ No newline at end of file diff --git a/tutti-service/src/test/resources/log4j.properties b/tutti-service/src/test/resources/log4j.properties index 92a6861..11a2599 100644 --- a/tutti-service/src/test/resources/log4j.properties +++ b/tutti-service/src/test/resources/log4j.properties @@ -32,4 +32,5 @@ log4j.logger.fr.ifremer.tutti=INFO log4j.logger.org.nuiton.i18n=ERROR #See https://forum.hibernate.org/viewtopic.php?p=2404391 +log4j.logger.org.hibernate.engine.internal.StatefulPersistenceContext=ERROR log4j.logger.org.hibernate.engine.StatefulPersistenceContext.ProxyWarnLog=ERROR diff --git a/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java b/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java index cf4aa9b..ed35109 100644 --- a/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java +++ b/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java @@ -119,9 +119,9 @@ public class ImportBigfinAction extends AbstractTuttiAction<SpeciesBatchUIModel, importResult.getNbFrequenciesImported(), importResult.getNbFrequenciesDeleted())); - if (!importResult.getErrors().isEmpty()) { + if (!importResult.getWarnings().isEmpty()) { StringBuilder sb = new StringBuilder(); - for (String s : importResult.getErrors()) { + for (String s : importResult.getWarnings()) { sb.append("<li>").append(s).append("</li>"); } displayWarningMessage( @@ -135,7 +135,7 @@ public class ImportBigfinAction extends AbstractTuttiAction<SpeciesBatchUIModel, } else { StringBuilder sb = new StringBuilder(); - for (String s : importResult.getFatalErrors()) { + for (String s : importResult.getErrors()) { sb.append("<li>").append(s).append("</li>"); } displayWarningMessage( -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.