This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository tutti. See http://git.codelutin.com/tutti.git commit 0da3fdade03312855120fc6084dce71fe62b7480 Author: Tony CHEMIT <chemit@codelutin.com> Date: Tue Apr 14 14:58:09 2015 +0200 arret de la validation si le nombre de lignes en erreur est trop grand --- .../java/fr/ifremer/tutti/service/csv/CsvComsumer.java | 4 ++-- .../genericformat/GenericFormatCsvFileResult.java | 17 ++++++++++++++++- .../genericformat/GenericFormatImportRequest.java | 4 ++++ .../importactions/ImportActionSupport.java | 13 +++++++++++++ .../importactions/ValidateAccidentalCatchAction.java | 11 ++++++++++- .../importactions/ValidateCatchAction.java | 11 ++++++++++- .../importactions/ValidateGearCaracteristicAction.java | 11 ++++++++++- .../ValidateIndividualObservationAction.java | 11 ++++++++++- .../importactions/ValidateMarineLitterAction.java | 11 ++++++++++- .../importactions/ValidateOperationAction.java | 11 ++++++++++- .../importactions/ValidateParameterAction.java | 11 ++++++++++- .../importactions/ValidateSurveyAction.java | 11 ++++++++++- .../resources/ftl/genericFormatValidateReport_fr.ftl | 3 +++ 13 files changed, 118 insertions(+), 11 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java index 1ae3331..ab4226e 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java @@ -146,8 +146,8 @@ public abstract class CsvComsumer<O, M extends AbstractTuttiImportExportModel<O> } - public boolean foundSomeErrors() { - return !rowsInError.isEmpty(); + public int getNbRowsInErrors() { + return rowsInError.size(); } public Map<Long, Set<String>> getRowsInError() { diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatCsvFileResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatCsvFileResult.java index 3dcb189..ea5f37e 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatCsvFileResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatCsvFileResult.java @@ -38,9 +38,16 @@ import java.util.Set; public class GenericFormatCsvFileResult extends GenericFormatFileResult { private static final long serialVersionUID = 1L; - + + /** + * Number of lines in file. + */ private int linesCount; + /** + * Is the file was fully loaded ? + */ + private boolean fullyLoaded = true; public GenericFormatCsvFileResult(String filename, boolean mandatory, boolean found) { super(filename, mandatory, found); @@ -54,6 +61,14 @@ public class GenericFormatCsvFileResult extends GenericFormatFileResult { this.linesCount = linesCount; } + public boolean isFullyLoaded() { + return fullyLoaded; + } + + public void setFullyLoaded(boolean fullyLoaded) { + this.fullyLoaded = fullyLoaded; + } + public void flushErrors(CsvComsumer<?, ?> consumer) { Map<Long, Set<String>> rowsInError = consumer.getRowsInError(); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java index 5b64a6e..c1a8419 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java @@ -179,6 +179,10 @@ public class GenericFormatImportRequest implements Serializable { return importConfiguration.getReportFile(); } + public int getMaximumRowsInErrorPerFile() { + return importConfiguration.getMaximumRowsInErrorPerFile(); + } + protected String getExistingCruiseId(Cruise importRowCruise) { String result = null; diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ImportActionSupport.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ImportActionSupport.java index 94477d8..95d808f 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ImportActionSupport.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ImportActionSupport.java @@ -24,7 +24,9 @@ package fr.ifremer.tutti.service.genericformat.importactions; * #L% */ +import fr.ifremer.tutti.service.csv.CsvComsumer; import fr.ifremer.tutti.service.genericformat.GenericFormatContextSupport; +import fr.ifremer.tutti.service.genericformat.GenericFormatCsvFileResult; /** * Created on 3/3/15. @@ -62,4 +64,15 @@ public abstract class ImportActionSupport { protected void skipExecute() { // by default do nothing } + + protected void flushConsumer(CsvComsumer consumer, GenericFormatCsvFileResult importFileResult) { + + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + + boolean fullyLoaded = consumer.getNbRowsInErrors() <= maximumRowsInErrorPerFile; + importFileResult.flushErrors(consumer); + importFileResult.setFullyLoaded(fullyLoaded); + + } + } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateAccidentalCatchAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateAccidentalCatchAction.java index 4417916..a4f9572 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateAccidentalCatchAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateAccidentalCatchAction.java @@ -66,6 +66,8 @@ public class ValidateAccidentalCatchAction extends ImportActionSupport { log.info("Validate accidentalCatch.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getAccidentalCatchFileResult(); try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches(false)) { for (ImportRow<AccidentalCatchRow> row : consumer) { @@ -78,9 +80,16 @@ public class ValidateAccidentalCatchAction extends ImportActionSupport { consumer.prepareRowForPersist(operationContext, row); } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close accidentalCatch.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateCatchAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateCatchAction.java index 2fda8f4..a02ec06 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateCatchAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateCatchAction.java @@ -66,6 +66,8 @@ public class ValidateCatchAction extends ImportActionSupport { log.info("Validate catch.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getCatchFileResult(); try (CsvConsumerForCatch consumer = importContext.loadCatches(false)) { for (ImportRow<CatchRow> row : consumer) { @@ -78,9 +80,16 @@ public class ValidateCatchAction extends ImportActionSupport { consumer.prepareRowForPersist(operationContext, row); } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close catch.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateGearCaracteristicAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateGearCaracteristicAction.java index dabd0cc..02f5568 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateGearCaracteristicAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateGearCaracteristicAction.java @@ -66,6 +66,8 @@ public class ValidateGearCaracteristicAction extends ImportActionSupport { log.info("Validate gearCaracteristics.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getGearCaracteristicFileResult(); try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics(false)) { for (ImportRow<GearCaracteristicRow> row : consumer) { @@ -80,9 +82,16 @@ public class ValidateGearCaracteristicAction extends ImportActionSupport { } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close gearCaracteristic.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateIndividualObservationAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateIndividualObservationAction.java index 6d35e1c..dd62f13 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateIndividualObservationAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateIndividualObservationAction.java @@ -67,6 +67,8 @@ public class ValidateIndividualObservationAction extends ImportActionSupport { log.info("Validate individualObservation.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getIndividualObservationFileResult(); try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations(false)) { for (ImportRow<IndividualObservationRow> row : consumer) { @@ -79,9 +81,16 @@ public class ValidateIndividualObservationAction extends ImportActionSupport { consumer.prepareRowForPersist(operationContext, row); } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close individualObservation.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateMarineLitterAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateMarineLitterAction.java index 0dd8656..f1addc4 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateMarineLitterAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateMarineLitterAction.java @@ -66,6 +66,8 @@ public class ValidateMarineLitterAction extends ImportActionSupport { log.info("Validate marineLitter.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getMarineLitterFileResult(); try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters(false)) { for (ImportRow<MarineLitterRow> row : consumer) { @@ -78,9 +80,16 @@ public class ValidateMarineLitterAction extends ImportActionSupport { consumer.prepareRowForPersist(operationContext, row); } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close marineLitter.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateOperationAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateOperationAction.java index ab1718f..1fe6d31 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateOperationAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateOperationAction.java @@ -72,6 +72,8 @@ public class ValidateOperationAction extends ImportActionSupport { boolean importBenthos = importContext.getImportRequest().isImportBenthos(); boolean importMarineLitter = importContext.getImportRequest().isImportMarineLitter(); + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getOperationFileResult(); try (CsvConsumerForOperation consumer = importContext.loadOperations(false)) { for (ImportRow<OperationRow> row : consumer) { @@ -93,9 +95,16 @@ public class ValidateOperationAction extends ImportActionSupport { } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close operation.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateParameterAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateParameterAction.java index 8bcb592..dc3c66f 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateParameterAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateParameterAction.java @@ -66,6 +66,8 @@ public class ValidateParameterAction extends ImportActionSupport { log.info("Validate parameter.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getParameterFileResult(); try (CsvConsumerForParameter consumer = importContext.loadParameters(false)) { for (ImportRow<ParameterRow> row : consumer) { @@ -78,9 +80,16 @@ public class ValidateParameterAction extends ImportActionSupport { consumer.prepareRowForPersist(operationContext, row); } + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close parameter.csv file", e); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateSurveyAction.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateSurveyAction.java index bf78825..b39d014 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateSurveyAction.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/importactions/ValidateSurveyAction.java @@ -75,6 +75,8 @@ public class ValidateSurveyAction extends ImportActionSupport { log.info("Validate survey.csv file."); } + int maximumRowsInErrorPerFile = importContext.getImportRequest().getMaximumRowsInErrorPerFile(); + GenericFormatCsvFileResult importFileResult = importContext.getSurveyFileResult(); try (CsvConsumerForSurvey consumer = importContext.loadSurveys(false)) { for (ImportRow<SurveyRow> row : consumer) { @@ -105,9 +107,16 @@ public class ValidateSurveyAction extends ImportActionSupport { importContext.addImportedCruise(cruise, null, existingCruiseData, existingFishingOperations); + if (consumer.getNbRowsInErrors() > maximumRowsInErrorPerFile) { + if (log.isWarnEnabled()) { + log.warn("Too much errors, stop validating this file."); + } + break; + } + } - importFileResult.flushErrors(consumer); + flushConsumer(consumer, importFileResult); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close survey.csv file", e); diff --git a/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl b/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl index 7448808..f76cb18 100644 --- a/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl +++ b/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl @@ -82,6 +82,9 @@ </#if> <#if errorsEntries?size != 0> <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> + <#if !fileResult.fullyLoaded> + <li class="error">Trop le lignes en erreurs, tout le fichier n'a pas été validé.</li> + </#if> </#if> </ul> <#if errorsEntries?size == 0> -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.