This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit d508d1fd915fdf4abcf5b53ca8e468d3372fe06d Author: Jean Couteau <jean.couteau@gmail.com> Date: Thu Mar 1 14:28:41 2018 +0100 Should fix biotic export --- .../service/atlantos/xml/XmlBioticExport.java | 373 ++++++++++++++------- ...yageCatchesBiometrySampleImportExportModel.java | 3 + 2 files changed, 253 insertions(+), 123 deletions(-) diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/atlantos/xml/XmlBioticExport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/atlantos/xml/XmlBioticExport.java index 70f81640..de6c91f1 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/atlantos/xml/XmlBioticExport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/atlantos/xml/XmlBioticExport.java @@ -9,6 +9,7 @@ import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.SampleDataTypeImpl; +import fr.ifremer.echobase.entities.references.Species; import fr.ifremer.echobase.entities.references.SpeciesCategory; import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.services.EchoBaseService; @@ -69,90 +70,218 @@ public class XmlBioticExport implements EchoBaseService { // EXPORT HAUL exportHaul(operation, index++, xmlCruise); - Map<String, Float> totalWeight = new HashMap<String, Float>(); - Map<String, Float> totalWeightBySize = new HashMap<String, Float>(); - Map<String, Integer> totalNumber = new HashMap<String, Integer>(); - Map<String, List<Sample>> individuals = new HashMap<String, List<Sample>>(); - - Collection<Sample> subSamples = new ArrayList<Sample>(); + //Map with all the totals with no subsamples - key : code#cat + Map<String,Sample> totals = new HashMap<>(); + + //Map with all the subsamples - key : code#cat + Map<String,List<Sample>> subsamples = new HashMap<>(); + + //Map with all the individuals - key : code#cat#lengthClass + Map<String,List<Sample>> individuals = new HashMap<>(); + + //Map with all the speciesCategoryWeight - key : code#cat + Map<String,Float> speciesCategoryWeights = new HashMap<>(); + + //Map with all the speciesCategoryNumber - key : code#cat + Map<String,Integer> speciesCategoryNumbers = new HashMap<>(); + + //Map with all the subsampledNumber - key : code#cat + Map<String,Integer> subsampledNumbers = new HashMap<>(); + + //Map with all the subsampledWeight - key : code#cat + Map<String,Float> subsampledWeights = new HashMap<>(); + Collection<Sample> samples = operation.getSample(); + + if (operation.getId().equals("T0211")){ + //System.out.println("Code#Cat : " + codeCatKey); + //System.out.println("LengthClass : " + sampleDataValues.get("LengthClass")); + System.out.println(samples.size()); + } + for (Sample sample : samples) { - + String name = sample.getSampleType().getName(); - - if ("Total".equals(name)) { - SpeciesCategory category = sample.getSpeciesCategory(); - String code = category.getSpecies().getBaracoudaCode(); - String size = code + "#" + category.getSizeCategoryLabel(); - - Float weight = totalWeight.get(code); - if (weight == null) { - weight = 0f; - } - - Float sampleWeight = sample.getSampleWeight(); - if (sampleWeight != null) { - weight += sampleWeight; - } - totalWeight.put(code, weight); - - Float weightBySize = totalWeightBySize.get(size); - if (weightBySize == null) { - weightBySize = 0f; - } - - if (sampleWeight != null) { - weightBySize += sampleWeight; - } - totalWeightBySize.put(size, weightBySize); - - Integer number = totalNumber.get(code); - if (number == null) { - number = 0; - } - - Integer numberSampled = sample.getNumberSampled(); - if (numberSampled != null) { - number += numberSampled; + + SpeciesCategory category = sample.getSpeciesCategory(); + + //FIXME jcouteau : pour les tests. En prod, il faut que ça pète si null -> données non valides alors que export de données valides uniquement + //if (category != null) { + + //Compute code#cat key + Species species = category.getSpecies(); + String code = species.getBaracoudaCode(); + String categoryName = ""; + if (category.getSizeCategory() != null) { + categoryName = category.getSizeCategory().getName(); } - totalNumber.put(code, number); - - subSamples.add(sample); - } - - if ("Subsample".equals(name)) { - subSamples.add(sample); - } - - if ("Individual".equals(name)) { - SpeciesCategory category = sample.getSpeciesCategory(); - String code = category.getSpecies().getBaracoudaCode(); - - List<Sample> individualSamples = individuals.get(code); - if (individualSamples == null) { - individualSamples = new ArrayList<Sample>(); - individuals.put(code, individualSamples); + String codeCatKey = code + "#" + categoryName; + + if ("Total".equals(name)) { + //Add sample to totals list if no subsamples (yet) + if (subsamples.get(codeCatKey) == null) { + totals.put(codeCatKey, sample); + } + + //Add sample weight to speciesCategoryWeight + Float weight = speciesCategoryWeights.get(codeCatKey); + if (weight == null) { + weight = 0f; + } + + Float sampleWeight = sample.getSampleWeight(); + if (sampleWeight != null) { + weight += sampleWeight; + } + speciesCategoryWeights.put(codeCatKey, weight); + + //Add sample number to speciesCategoryNumber + Integer number = speciesCategoryNumbers.get(codeCatKey); + if (number == null) { + number = 0; + } + + Integer numberSampled = sample.getNumberSampled(); + if (numberSampled != null) { + number += numberSampled; + } + speciesCategoryNumbers.put(codeCatKey, number); + + } else if ("Subsample".equals(name)) { + + //Get back sample datas + Map<String, String> sampleDataValues = getSampleDataValues(sample); + + // get back weight at length for this subsample + String weightAtLength = sampleDataValues.get(SampleDataTypeImpl.WEIGHT_AT_LENGTHKG); + Float sampleWeight = null; + if (weightAtLength != null) { + sampleWeight = Float.parseFloat(weightAtLength); + } + + //add weightAtLength to the sum for this category + Float weightBySize = subsampledWeights.get(codeCatKey); + if (weightBySize == null) { + weightBySize = 0f; + } + if (sampleWeight != null) { + weightBySize += sampleWeight; + } + subsampledWeights.put(codeCatKey, weightBySize); + + //get back number at length for this subsample + String numberAtLength = sampleDataValues.get(SampleDataTypeImpl.NUMBER_AT_LENGTH); + Float sampleNumber = null; + if (numberAtLength != null) { + sampleNumber = Float.parseFloat(numberAtLength); + } + + //add numberAtLength to the sum for this category + Integer sumNumberAtLength = subsampledNumbers.get(codeCatKey); + if (sumNumberAtLength == null) { + sumNumberAtLength = 0; + } + if (sampleNumber != null) { + sumNumberAtLength += sampleNumber.intValue(); + } + subsampledNumbers.put(codeCatKey, sumNumberAtLength); + + //Add subsample to the list + List<Sample> speciesCategorySubsamples = subsamples.get(codeCatKey); + if (speciesCategorySubsamples == null) { + speciesCategorySubsamples = new ArrayList<>(); + subsamples.put(codeCatKey, speciesCategorySubsamples); + } + speciesCategorySubsamples.add(sample); + + //check if total present, if so, removes it + if (subsamples.get(codeCatKey) != null) { + totals.remove(codeCatKey); + } + + } else if ("Individual".equals(name)) { + + //Get back sample datas + Map<String, String> sampleDataValues = getSampleDataValues(sample); + + //Get back lengthclass for individual, remove trailing .0 + String lengthClass = sampleDataValues.get("LTmm1"); + lengthClass = lengthClass.substring(0,lengthClass.indexOf(".")); + + String codeCatLCkey = codeCatKey+"#"+lengthClass; + + //add individual to the list + List<Sample> lengthClassIndividuals = individuals.get(codeCatLCkey); + if (lengthClassIndividuals == null) { + lengthClassIndividuals = new ArrayList<>(); + individuals.put(codeCatLCkey, lengthClassIndividuals); + } + lengthClassIndividuals.add(sample); + } - - individualSamples.add(sample); + //} + } + + //Export totals + for (Sample exportSample:totals.values()) { + + //Compute code#cat key + SpeciesCategory category = exportSample.getSpeciesCategory(); + Species species = category.getSpecies(); + String code = species.getBaracoudaCode(); + String categoryName = ""; + if (category.getSizeCategory() != null) { + categoryName = category.getSizeCategory().getName(); } + String codeCatKey = code + "#" + categoryName; + + //get speciesCategoryWeight and speciesCategoryNumber + Float speciesCategoryWeight = speciesCategoryWeights.get(codeCatKey); + Integer speciesCategoryNumber = speciesCategoryNumbers.get(codeCatKey); + + //Export Catch - Total sample so no subsampledWeight or subsampledNumber + exportCatch(exportSample,null,speciesCategoryWeight,null,speciesCategoryNumber,xmlCruise); + + xmlCruise.close("Catch"); } - - for (Sample sample : subSamples) { - SpeciesCategory category = sample.getSpeciesCategory(); - //export only ICES allowed species - //if (category.getSpecies().getIcesExport() != null && category.getSpecies().getIcesExport()) { - String code = category.getSpecies().getBaracoudaCode(); - String size = code + "#" + category.getSizeCategoryLabel(); - Float weight = totalWeight.get(code); - Float weightBySize = totalWeightBySize.get(size); - Integer number = totalNumber.get(code); + //Export subsamples + for (List<Sample> exportSubsamples:subsamples.values()){ + for (Sample exportSample : exportSubsamples){ + //Compute code#cat key + SpeciesCategory category = exportSample.getSpeciesCategory(); + Species species = category.getSpecies(); + String code = species.getBaracoudaCode(); + String categoryName = ""; + if (category.getSizeCategory() != null) { + categoryName = category.getSizeCategory().getName(); + } + String codeCatKey = code + "#" + categoryName; + + //Compute code#cat#lengthClass key + Map<String, String> sampleDataValues = getSampleDataValues(exportSample); + + //Get back lengthclass for individual + String lengthClass = sampleDataValues.get("LengthClass"); + Float lengthClassValue = 0f; + if (lengthClass != null) { + lengthClassValue = Float.parseFloat(lengthClass) * 10; + } + String codeCatLCkey = codeCatKey+"#"+lengthClassValue.intValue(); - // EXPORT CATCH - exportCatch(sample, weightBySize, weight, number, xmlCruise); + //get speciesCategoryWeight and speciesCategoryNumber + Float speciesCategoryWeight = speciesCategoryWeights.get(codeCatKey); + Integer speciesCategoryNumber = speciesCategoryNumbers.get(codeCatKey); - List<Sample> individualSamples = individuals.remove(code); + //get subSampledWeight and subsampledNumbers + Float subsampledWeight = subsampledWeights.get(codeCatKey); + Integer subsampledNumber = subsampledNumbers.get(codeCatKey); + + //Export Catch + exportCatch(exportSample,subsampledWeight,speciesCategoryWeight,subsampledNumber,speciesCategoryNumber,xmlCruise); + + //Export individuals (only individuals for this subsample) + List<Sample> individualSamples = individuals.get(codeCatLCkey); if (individualSamples != null) { for (Sample individualSample : individualSamples) { // EXPORT BIOLOGY @@ -161,7 +290,7 @@ public class XmlBioticExport implements EchoBaseService { } xmlCruise.close("Catch"); - //} + } } xmlCruise.close("Haul"); @@ -171,7 +300,7 @@ public class XmlBioticExport implements EchoBaseService { xmlCruise.close("Biotic"); vocabulary.generate(); } - + public void exportCruise(Voyage voyage, Vessel vessel, XmlWriter xml) throws IOException { xml.open("Cruise"); @@ -307,28 +436,10 @@ public class XmlBioticExport implements EchoBaseService { // xml.create("LogDistance"); } - public void exportCatch(Sample subsample, Float weightBySize, Float weight, Integer number, XmlWriter xml) throws IOException { + public void exportCatch(Sample subsample, Float subsampledWeight, Float speciesCategoryWeight, Integer subsampledNumber, Integer speciesCategoryNumber, XmlWriter xml) throws IOException { xml.open("Catch"); - - Map<String, String> sampleDataValues = new HashMap<String, String>(); - Collection<SampleData> datas = subsample.getSampleData(); - for (SampleData data : datas) { - - String name = data.getSampleDataType().getName(); - Float value = data.getDataValue(); - String label = data.getDataLabel(); - - if (SampleDataTypeImpl.LTCM0_5.equals(name) - || SampleDataTypeImpl.L_TCM_1.equals(name)) { - - sampleDataValues.put(SampleDataTypeImpl.NUMBER_AT_LENGTH, String.valueOf(value)); - sampleDataValues.put(name, label); - sampleDataValues.put("LengthClass", label); - - } else { - sampleDataValues.put(name, String.valueOf(value)); - } - } + + Map<String, String> sampleDataValues = getSampleDataValues(subsample); SpeciesCategory category = subsample.getSpeciesCategory(); @@ -337,16 +448,11 @@ public class XmlBioticExport implements EchoBaseService { if (lengthClass != null) { lengthClassValue = Float.parseFloat(lengthClass) * 10; } - + String numberAtLength = sampleDataValues.get(SampleDataTypeImpl.NUMBER_AT_LENGTH); String weightAtLength = sampleDataValues.get(SampleDataTypeImpl.WEIGHT_AT_LENGTHKG); - - double subSamplingFactor = 0; - if (weightBySize != null){ - subSamplingFactor = weightBySize / subsample.getSampleWeight(); - } - - xml.create("DataType", + + xml.create("DataType", "IDREF", vocabulary.getVocabularyCode("AC_CatchDataType_R")); xml.create("SpeciesCode", "IDREF", vocabulary.getVocabularyCode("SpecWoRMS_" + category.getSpecies().getWormsCode())); @@ -361,30 +467,28 @@ public class XmlBioticExport implements EchoBaseService { 2); } xml.create("SpeciesCategoryNumber", - number); + speciesCategoryNumber); xml.create("WeightUnit", "IDREF", vocabulary.getVocabularyCode("AC_WeightUnit_kg")); xml.create("SpeciesCategoryWeight", - weight != null ? weight : 0); - - /*xml.create("SpeciesCategory", - "IDREF", vocabulary.getVocabularyCode(category.getSizeCategoryLabel(), "AC_CatchCategory_1")); */ + speciesCategoryWeight); xml.create("SpeciesSex"); - xml.create("SubsampledNumber", - subsample.getNumberSampled()); - xml.create("SubsamplingFactor", - subSamplingFactor); - - if ((int)subSamplingFactor==1) { - xml.create("SubsampleWeight", - weight != null ? weight.intValue() : 0); - } else { + + if (subsampledNumber != null && subsampledNumber != 0) { + xml.create("SubsampledNumber", + subsampledNumber); + if (weightAtLength != null && subsampledWeight != null) { + xml.create("SubsamplingFactor", + Float.parseFloat(weightAtLength) / subsampledWeight); + } else { + xml.create("SubsamplingFactor", + 0); + } xml.create("SubsampleWeight", - subsample.getSampleWeight()); - + subsampledWeight); } - if (lengthClassValue.intValue()!= 0) { + if (lengthClassValue!= 0) { xml.create("LengthCode", "IDREF", vocabulary.getVocabularyCode("AC_LengthCode_mm")); xml.create("LengthClass", @@ -392,7 +496,7 @@ public class XmlBioticExport implements EchoBaseService { xml.create("LengthType", "IDREF", vocabulary.getVocabularyCode("AC_LengthMeasurementType_1")); xml.create("NumberAtLength", - subsample.getNumberSampled()); + numberAtLength); xml.create("WeightAtLength", weightAtLength != null ? weightAtLength : 0); } @@ -449,4 +553,27 @@ public class XmlBioticExport implements EchoBaseService { xml.close("Biology"); } + + protected Map<String, String> getSampleDataValues(Sample sample) { + Map<String, String> sampleDataValues = new HashMap<>(); + Collection<SampleData> datas = sample.getSampleData(); + for (SampleData data : datas) { + + String dataName = data.getSampleDataType().getName(); + Float value = data.getDataValue(); + String label = data.getDataLabel(); + + if (SampleDataTypeImpl.LTCM0_5.equals(dataName) + || SampleDataTypeImpl.L_TCM_1.equals(dataName)) { + + sampleDataValues.put(SampleDataTypeImpl.NUMBER_AT_LENGTH, String.valueOf(value)); + sampleDataValues.put(dataName, label); + sampleDataValues.put("LengthClass", label); + + } else { + sampleDataValues.put(dataName, String.valueOf(value)); + } + } + return sampleDataValues; + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportExportModel.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportExportModel.java index 336ef4b2..922e54a7 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportExportModel.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportExportModel.java @@ -24,6 +24,7 @@ import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; import fr.ifremer.echobase.entities.references.SampleDataType; +import fr.ifremer.echobase.entities.references.SizeCategory; import fr.ifremer.echobase.entities.references.Species; import fr.ifremer.echobase.services.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.services.service.importdata.contexts.VoyageCatchesImportDataContext; @@ -44,6 +45,7 @@ public class VoyageCatchesBiometrySampleImportExportModel extends EchoBaseImport VoyageCatchesBiometrySampleImportExportModel model = new VoyageCatchesBiometrySampleImportExportModel(importDataContext.getCsvSeparator()); model.newForeignKeyColumn(EchoBaseCsvUtil.OPERATION_ID, VoyageCatchesBiometrySampleImportRow.PROPERTY_OPERATION, Operation.class, Operation.PROPERTY_ID, importDataContext.getVoyageOperationsById()); model.newForeignKeyColumn(Species.PROPERTY_BARACOUDA_CODE, VoyageCatchesBiometrySampleImportRow.PROPERTY_SPECIES, Species.class, Species.PROPERTY_BARACOUDA_CODE, importDataContext.getSpeciesByBaracoudaCode()); + model.newForeignKeyColumn(VoyageCatchesSubSampleImportRow.PROPERTY_SIZE_CATEGORY, SizeCategory.class, SizeCategory.PROPERTY_NAME, importDataContext.getSizeCategoriesByName()); model.newMandatoryColumn(VoyageCatchesBiometrySampleImportRow.PROPERTY_NUM_FISH, EchoBaseCsvUtil.PRIMITIVE_INTEGER); @@ -57,6 +59,7 @@ public class VoyageCatchesBiometrySampleImportExportModel extends EchoBaseImport VoyageCatchesBiometrySampleImportExportModel model = new VoyageCatchesBiometrySampleImportExportModel(importDataContext.getCsvSeparator()); model.newColumnForExport(EchoBaseCsvUtil.OPERATION_ID, VoyageCatchesBiometrySampleImportRow.PROPERTY_OPERATION, EchoBaseCsvUtil.OPERATION_FORMATTER); model.newColumnForExport(Species.PROPERTY_BARACOUDA_CODE, VoyageCatchesBiometrySampleImportRow.PROPERTY_SPECIES, EchoBaseCsvUtil.SPECIES_FORMATTER); + model.newColumnForExport(VoyageCatchesSubSampleImportRow.PROPERTY_SEX_CATEGORY, EchoBaseCsvUtil.SEX_CATEGORY_FORMATTER); model.newColumnForExport(VoyageCatchesBiometrySampleImportRow.PROPERTY_NUM_FISH, EchoBaseCsvUtil.PRIMITIVE_INTEGER); model.newColumnForExport("name", SampleData.PROPERTY_SAMPLE_DATA_TYPE, EchoBaseCsvUtil.SAMPLE_DATA_TYPE_FORMATTER); model.newColumnForExport(SampleData.PROPERTY_DATA_LABEL); -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.