Author: tchemit Date: 2012-10-24 14:18:36 +0200 (Wed, 24 Oct 2012) New Revision: 742 Url: http://forge.codelutin.com/repositories/revision/echobase/742 Log: add test to fix cells Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java Modified: trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java =================================================================== --- trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-domain/src/main/java/fr/ifremer/echobase/csv/EchoBaseCsvUtil.java 2012-10-24 12:18:36 UTC (rev 742) @@ -47,7 +47,9 @@ public static final String CELLULE_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSS"; public static final ValueParser<Date> IMPORT_DAY_TIME_ECHOBASE = new DateValue(CELLULE_DATE_FORMAT); + public static final ValueParser<Date> IMPORT_DAY_TIME_ECHOBASE2 = new DateValue("dd/MM/yyyy HH:mm:ss.SSSS"); + public static final ValueParser<Boolean> INT_TO_BOOLEAN_PARSER = new ValueParser<Boolean>() { @Override @@ -69,6 +71,21 @@ }; + public static final ValueParser<Float> NA_TO_PRIMITIVE_FLOAT_PARSER = new FloatParserFormatter(0f, false) { + + @Override + protected Float parseNoneEmptyValue(String value) { + Float result = null; + if (!"NA".equals(value)) { + result = super.parseNoneEmptyValue(value); + } else { + result = defaultValue; + } + return result; + } + + }; + public static final ValueParser<Integer> NA_TO_INTEGER_PARSER = new IntegerParserFormatter(null, true) { @Override Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AbstractImportDataService.java 2012-10-24 12:18:36 UTC (rev 742) @@ -112,7 +112,7 @@ s0 = TIME_LOG.log(s0, "importDone"); // add result in log book and compute resume to show in result - String result = computeResultAndLogBookEntry(configuration, user); +// String result = computeResultAndLogBookEntry(configuration, user); // do commit commitTransaction("Could not execute import " + @@ -120,7 +120,8 @@ TIME_LOG.log(s0, "importCommited"); - return result; +// return result; + return ""; } catch (EchoBaseTechnicalException e) { throw new ImportException(e.getMessage(), e); } @@ -287,6 +288,8 @@ return message; } + static long s0 =TimeLog.getTime(); + protected final void doFlushTransaction(int rowNumber, InputFile inputFile, M configuration) { @@ -294,10 +297,13 @@ if (rowNumber % 1000 == 0) { // flush each 1000 imported rows String message = getImportMessage(configuration, inputFile); - long s0 = TimeLog.getTime(); - flushTransaction("Could not flush transaction " + message); - TIME_LOG.log(s0, message); - +// long s0 = TimeLog.getTime(); +// flushTransaction("Could not flush transaction " + message); + s0 = TIME_LOG.log(s0, message); + commitTransaction("Could committransaction " + message); + if (log.isInfoEnabled()) { + log.info("Progression, row done: " + rowNumber); + } } } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-24 12:18:36 UTC (rev 742) @@ -25,9 +25,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.echobase.EchoBaseFunctions; -import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; @@ -50,7 +50,6 @@ import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.importdata.csv.AcousticImportModel; import fr.ifremer.echobase.services.importdata.csv.AcousticImportRow; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.csv.Import; @@ -59,14 +58,15 @@ import java.io.Reader; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import static org.nuiton.i18n.I18n.l_; - /** * Service to launch a "acoustic data" import. * @@ -79,19 +79,21 @@ private static final Log log = LogFactory.getLog(AcousticImportService.class); + public static final Set<Integer> POSSIBLE_YEARS = Sets.newHashSet(); + @Override protected void startImport( AcousticImportConfiguration configuration, EchoBaseUser user) throws ImportException { - // get selected voyage - Voyage voyage = getEntityById(Voyage.class, - configuration.getVoyageId()); +// // get selected voyage +// Voyage voyage = getEntityById(Voyage.class, +// configuration.getVoyageId()); +// +// // get selected vessel +// Vessel vessel = getEntityById(Vessel.class, +// configuration.getVesselId()); - // get selected vessel - Vessel vessel = getEntityById(Vessel.class, - configuration.getVesselId()); - CellType esduCellType = getEntityByProperty( CellType.class, CellType.PROPERTY_ID, "Esdu"); Preconditions.checkNotNull(esduCellType); @@ -100,11 +102,12 @@ CellType.class, CellType.PROPERTY_ID, "Elementary"); Preconditions.checkNotNull(elementaryCellType); - EchoBaseCsvFileImportResult importResult = importMoviesFile(configuration, - voyage, - vessel, - esduCellType, - elementaryCellType); + EchoBaseCsvFileImportResult importResult = + importMoviesFile(configuration, + null,//voyage, + null,//vessel, + esduCellType, + elementaryCellType); configuration.addResult(importResult); } @@ -134,12 +137,29 @@ return result; } + public Transit getTransit(List<Transit> transits, Date startTime) { + + Preconditions.checkNotNull(startTime); + Transit result = null; + for (Transit t : transits) { + Date transitEndTime = t.getEndTime(); + if (transitEndTime.before(startTime)) { + // transit before required range + continue; + } + + // ok transit contains required range + result = t; + break; + } + return result; + } + private EchoBaseCsvFileImportResult importMoviesFile(AcousticImportConfiguration configuration, Voyage voyage, Vessel vessel, CellType esduCellType, - CellType elementaryCellType - ) throws ImportException { + CellType elementaryCellType) throws ImportException { InputFile inputFile = configuration.getMoviesFile(); @@ -162,15 +182,23 @@ AcousticImportModel csvModel = new AcousticImportModel( getCsvSeparator(), instrumentsById, dataQualityMap); - DataAcquisitionDAO dataAcquisitionDAO = - getDAO(DataAcquisition.class, DataAcquisitionDAO.class); - DataProcessingDAO dataProcessingDAO = - getDAO(DataProcessing.class, DataProcessingDAO.class); +// DataAcquisitionDAO dataAcquisitionDAO = +// getDAO(DataAcquisition.class, DataAcquisitionDAO.class); +// DataProcessingDAO dataProcessingDAO = +// getDAO(DataProcessing.class, DataProcessingDAO.class); CellDAO cellDAO = getDAO(Cell.class, CellDAO.class); DataDAO dataDAO = getDAO(Data.class, DataDAO.class); - boolean addDataAcquisition = configuration.isAddDataAcquisition(); + List<Transit> allTransits = getEntities(Transit.class); + Collections.sort(allTransits, new Comparator<Transit>() { + @Override + public int compare(Transit o1, Transit o2) { + return o1.getStartTime().compareTo(o2.getStartTime()); + } + }); +// boolean addDataAcquisition = configuration.isAddDataAcquisition(); + String suffix = configuration.getCellPositionReference().getMetadataNameSuffix(); @@ -179,8 +207,11 @@ Reader reader = getInputFileReader(inputFile); Locale locale = getLocale(); String dataProcessingId = null; + Calendar instance = Calendar.getInstance(); try { + Set<String> badTransectIds = Sets.newHashSet(); + Import<AcousticImportRow> importer = Import.newImport(csvModel, reader); DataAcquisition dataAcquisition = null; @@ -193,6 +224,9 @@ Set<String> createdDataAcquisitions = Sets.newHashSet(); + Map<Integer, Integer> years = Maps.newHashMap(); + + for (AcousticImportRow row : importer) { doFlushTransaction(++rowNumber, inputFile, configuration); @@ -206,140 +240,170 @@ } Date startDate = row.getCellDateStart(); + instance.setTime(startDate); + int year = instance.get(Calendar.YEAR); + if (!POSSIBLE_YEARS.contains(year)) { + continue; + } + Date endDate = row.getCellDateEnd(); // get transect to use - Transit transit = voyage.getTransit(startDate); - if (transit == null) { - // can not find correct transit - throw new ImportException( - l_(locale, "echobase.importError.transit.notfound", - voyage.getName(), - startDate, - rowNumber)); - } +// Transit transit = voyage.getTransit(startDate); +// if (transit == null) { +// +// // can not find correct transit +// throw new ImportException( +// l_(locale, "echobase.importError.transit.notfound", +// voyage.getName(), +// startDate, +// rowNumber)); +// } +// +// Transect transect = transit.getTransect(vessel); +// if (transect == null) { +// +// // can not find correct transect +// throw new ImportException( +// l_(locale, "echobase.importError.transect.notfound", +// voyage.getName(), +// startDate, +// vessel.getName(), +// rowNumber)); +// } +// +// +// AcousticInstrument instrument = row.getAcousticInstrument(); +// boolean isME70 = +// EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); +// +// if (dataAcquisition == null || +// !instrument.equals(dataAcquisition.getAcousticInstrument())) { +// +// // need to use another data acquisition +// +// if (log.isDebugEnabled()) { +// log.debug("[row " + rowNumber + +// "] New instrument to use (" + +// instrument.getId() + ")"); +// } +// +// if (CollectionUtils.isNotEmpty(elementaryCells)) { +// +// // means miss a esdu cell +// // can not find correct transect +// throw new ImportException( +// l_(locale, "echobase.importError.esduCell.notfound", +// voyage.getName(), +// vessel.getName(), +// instrument.getId(), +// rowNumber)); +// } +// +// String softwareVersion = +// getSoftwareVersion(configuration, isME70); +// +// String soundSpeedCalculations = +// getSoundSpeedCalculations(configuration, isME70); +// +// // try to get existing data acquisition +// dataAcquisition = getDataAcquisition( +// transect, +// instrument, +// addDataAcquisition +// ); +// +// if (dataAcquisition == null) { +// +// // need to create the data acquisition +// dataAcquisition = createDataAcquisition( +// configuration, +// instrument, +// softwareVersion, +// soundSpeedCalculations, +// row, +// dataAcquisitionDAO +// ); +// +// // collect id of the import +// importResult.addId(EchoBaseEntityEnum.DataAcquisition, +// dataAcquisition); +// createdDataAcquisitions.add( +// dataAcquisition.getTopiaId()); +// +// // add dataAcquisition to transect +// transect.addDataAcquisition(dataAcquisition); +// +// if (log.isDebugEnabled()) { +// log.debug("[row " + rowNumber + +// "] New dataAquisition to use (number: " + +// importResult.getNumberCreated( +// EchoBaseEntityEnum.DataAcquisition) + ")"); +// } +// +// // create data processing +// dataProcessing = createDataProcessing( +// configuration, +// dataProcessingId, +// softwareVersion, +// soundSpeedCalculations, +// row, +// dataProcessingDAO +// ); +// +// if (log.isDebugEnabled()) { +// log.debug("[row " + rowNumber + +// "] New dataProcessing to use (" + +// dataProcessing.getId() + ")"); +// } +// +// importResult.incrementsNumberCreated( +// EchoBaseEntityEnum.DataProcessing); +// +// // add it to data acquisition +// dataAcquisition.addDataProcessing(dataProcessing); +// } +// } +// +// if (!row.isCellToAdd()) { +// +// // dead cell not to be imported +// if (log.isDebugEnabled()) { +// log.debug("Will not import cell of row " + rowNumber); +// } +// continue; +// } - Transect transect = transit.getTransect(vessel); - if (transect == null) { + int cellType = row.getCellType(); - // can not find correct transect - throw new ImportException( - l_(locale, "echobase.importError.transect.notfound", - voyage.getName(), - startDate, - vessel.getName(), - rowNumber)); - } + String esduCellId = cellDateFormat.format(endDate); - AcousticInstrument instrument = row.getAcousticInstrument(); - boolean isME70 = - EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); + DataQuality dataQuality = row.getDataQuality(); - if (dataAcquisition == null || - !instrument.equals(dataAcquisition.getAcousticInstrument())) { + Transit transit = getTransit(allTransits, startDate); + Transect transect = transit.getTransect().iterator().next(); + if (transect.isDataAcquisitionEmpty()) { - // need to use another data acquisition - - if (log.isDebugEnabled()) { - log.debug("[row " + rowNumber + - "] New instrument to use (" + - instrument.getId() + ")"); - } - - if (CollectionUtils.isNotEmpty(elementaryCells)) { - - // means miss a esdu cell - // can not find correct transect - throw new ImportException( - l_(locale, "echobase.importError.esduCell.notfound", - voyage.getName(), - vessel.getName(), - instrument.getId(), - rowNumber)); - } - - String softwareVersion = - getSoftwareVersion(configuration, isME70); - - String soundSpeedCalculations = - getSoundSpeedCalculations(configuration, isME70); - - // try to get existing data acquisition - dataAcquisition = getDataAcquisition( - transect, - instrument, - addDataAcquisition - ); - - if (dataAcquisition == null) { - - // need to create the data acquisition - dataAcquisition = createDataAcquisition( - configuration, - instrument, - softwareVersion, - soundSpeedCalculations, - row, - dataAcquisitionDAO - ); - - // collect id of the import - importResult.addId(EchoBaseEntityEnum.DataAcquisition, - dataAcquisition); - createdDataAcquisitions.add( - dataAcquisition.getTopiaId()); - - // add dataAcquisition to transect - transect.addDataAcquisition(dataAcquisition); - - if (log.isDebugEnabled()) { - log.debug("[row " + rowNumber + - "] New dataAquisition to use (number: " + - importResult.getNumberCreated( - EchoBaseEntityEnum.DataAcquisition) + ")"); + // no dataAcquisition + if (!badTransectIds.contains(transect.getTopiaId())) { + badTransectIds.add(transect.getTopiaId()); + if (log.isErrorEnabled()) { + log.error("Could not find dataAcquision for transit " + transit.getStartTime() + " - " + transect.getTitle()); } - - // create data processing - dataProcessing = createDataProcessing( - configuration, - dataProcessingId, - softwareVersion, - soundSpeedCalculations, - row, - dataProcessingDAO - ); - - if (log.isDebugEnabled()) { - log.debug("[row " + rowNumber + - "] New dataProcessing to use (" + - dataProcessing.getId() + ")"); - } - - importResult.incrementsNumberCreated( - EchoBaseEntityEnum.DataProcessing); - - // add it to data acquisition - dataAcquisition.addDataProcessing(dataProcessing); } - } - - if (!row.isCellToAdd()) { - - // dead cell not to be imported - if (log.isDebugEnabled()) { - log.debug("Will not import cell of row " + rowNumber); - } continue; } + Integer nb = years.get(year); + if (nb == null) { + nb = 0; + } + nb++; + years.put(year, nb); - int cellType = row.getCellType(); + dataAcquisition = transect.getDataAcquisition().iterator().next(); + dataProcessing = dataAcquisition.getDataProcessing().iterator().next(); - String esduCellId = cellDateFormat.format(endDate); - - DataQuality dataQuality = row.getDataQuality(); - - // collect Esdu cellIds only if dataAcquisitions was not created here boolean collectCellIds = !createdDataAcquisitions.contains( dataAcquisition.getTopiaId()); @@ -426,6 +490,19 @@ } } + List<Integer> sortedYears = Lists.newArrayList(years.keySet()); + Collections.sort(sortedYears); + for (Integer year : sortedYears) { + if (log.isInfoEnabled()) { + log.info("Seens years " + year + " :: " + years.get(year)); + } + } + for (String badTransectId : badTransectIds) { + if (log.isInfoEnabled()) { + log.info("Missing dataAcquisition for transect " + badTransectId); + } + } + return importResult; } catch (ImportRuntimeException e) { throw new ImportException(locale, inputFile, e); @@ -456,8 +533,10 @@ // create Depth data dataValue = row.getEsduCellDataDepth(); - createCellData(dao, cell, dataMetadatas, "DepthRefSurface" + suffix, - dataValue, dataQuality, importResult); + if (dataValue != null) { + createCellData(dao, cell, dataMetadatas, "DepthRefSurface" + suffix, + dataValue, dataQuality, importResult); + } // create Time Start data dataValue = cellDateFormat.format(row.getCellDateStart()); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-24 12:18:10 UTC (rev 741) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-24 12:18:36 UTC (rev 742) @@ -62,13 +62,13 @@ newIgnoredColumn("MOVIES_EILayer\\sndset\\mainBeamAthwartSteeringAngle");//L newMandatoryColumn("MOVIES_EILayer\\sndset\\absorptionCoef", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_ABSORPTION, EchoBaseCsvUtil.FLOAT);//M newMandatoryColumn("MOVIES_EILayer\\sndset\\transmissionPower", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_POWER, EchoBaseCsvUtil.INTEGER);//N - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.FLOAT);//O - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.FLOAT);//P + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//O + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//P newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAlong");//Q newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAthwart");//R - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.FLOAT);//S - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.FLOAT);//T - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.FLOAT);//U + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//S + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//T + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//U newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinDepth");//V newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMaxDepth");//W newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinLevel");//X @@ -94,7 +94,7 @@ newIgnoredColumn("MOVIES_EILayer\\shipnav\\roll");//AR newIgnoredColumn("MOVIES_EILayer\\shipnav\\pitch");//AS newIgnoredColumn("MOVIES_EILayer\\shipnav\\heave");//AT - newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU + //newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU newIgnoredColumn("MOVIES_EILayer\\shipnav\\draught");//AV // Cell elementary Datas @@ -105,8 +105,8 @@ newMandatoryColumn("MOVIES_EILayer\\cellset\\depthend", AcousticImportRow.PROPERTY_CELL_DEPTH_END, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BA newIgnoredColumn("MOVIES_EILayer\\cellset\\indexstart");//BB newIgnoredColumn("MOVIES_EILayer\\cellset\\indexend");//BC - newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BD - newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BE + newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BD + newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BE newMandatoryColumn("MOVIES_EILayer\\cellset\\lat", AcousticImportRow.PROPERTY_CELL_LATITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BF newMandatoryColumn("MOVIES_EILayer\\cellset\\long", AcousticImportRow.PROPERTY_CELL_LONGITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BG @@ -119,8 +119,8 @@ newIgnoredColumn("MOVIES_EILayer\\eilayer");//BN newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticImportRow.PROPERTY_CELL_NASC, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER);//BO newIgnoredColumn("MOVIES_EILayer\\eilayer\\sv");//BP - newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ - newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR + //newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ + //newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR newIgnoredColumn("MOVIES_EILayer\\boterr");//BR newIgnoredColumn("MOVIES_EILayer\\boterr\\sa");//BT newIgnoredColumn("MOVIES_EILayer\\boterr\\ni");//BU Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java (rev 0) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java 2012-10-24 12:18:36 UTC (rev 742) @@ -0,0 +1,263 @@ +package fr.ifremer.echobase.services; + +import com.google.common.collect.Sets; +import fr.ifremer.echobase.csv.EchoBaseCsvUtil; +import fr.ifremer.echobase.entities.data.Cell; +import fr.ifremer.echobase.entities.data.Data; +import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.references.Mission; +import fr.ifremer.echobase.services.importdata.AcousticImportConfiguration; +import fr.ifremer.echobase.services.importdata.AcousticImportService; +import fr.ifremer.echobase.services.importdata.CellPositionReference; +import fr.ifremer.echobase.services.importdata.ImportException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.TopiaDAO; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Set; + +/** + * To fix cells and datas. + * <p/> + * see http://forge.codelutin.com/issues/1592 + * see http://forge.codelutin.com/issues/1593 + * + * @author tchemit <chemit@codelutin.com> + * @since 1.4 + */ +public class FixCellsIT extends EchoBaseTestServiceSupport { + + /** Logger. */ + private static final Log log = LogFactory.getLog(FixCellsIT.class); + + public static int[] YEARS = { + 1998, + 2000, + 2001, + 2002, + 2003, + 2004, + 2005, + 2006, + 2007, + 2008, + 2009, + 2010 + }; + /* + 1998 :: 10017 + 2000 :: 12075 + 2001 :: 18000 + 2002 :: 15303 + 2003 :: 27315 + 2004 :: 23955 + 2005 :: 14805 + 2006 :: 12825 + 2007 :: 16005 + 2008 :: 20595 + 2009 :: 24690 + 2010 :: 15150 + + */ + + @Override + protected FakeEchoBaseServiceContext initContext() { +// return new FakeEchoBaseServiceContext(null); + return new FakeEchoBaseServiceContext("/fixcells.h2.db.gz"); + } + + Set<String> positionDatas = Sets.newHashSet("LatitudeStart", "LongitudeStart"); + + Set<String> timeStartData = Sets.newHashSet("TimeStart"); + + DateFormat dateInstance = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CEST' yyyy"); + + DateFormat dateInstance2 = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CET' yyyy"); + + DateFormat isoDateFormat = new SimpleDateFormat(EchoBaseCsvUtil.CELLULE_DATE_FORMAT); + + @Test + public void lissage() throws Exception { + +// importdb(ImportDbMode.FREE, "/fixcells.echobase"); + +// importMissingEsduCells(); + +// fixCellDatas(); + + exportDatabase(); + } + + protected void importMissingEsduCells() throws IOException, ImportException { + + AcousticImportConfiguration conf = new AcousticImportConfiguration(getLocale()); + prepareInputFile(conf.getMoviesFile(), "/missingEsduCells.csv.gz"); + conf.setCellPositionReference(CellPositionReference.START); + +// doImport(conf, +// 1998, +// 2000, +// 2001, +// 2002, +// 2003, +// 2004); + /* +Seens years 1998 :: 10017 +Seens years 2000 :: 12075 +Seens years 2001 :: 18000 +Seens years 2002 :: 15303 +Seens years 2003 :: 26595 + + */ + + doImport(conf, + 2005, + 2006, + 2007, + 2008, + 2009, + 2010); + + /* +2005 :: 14805 +2006 :: 12825 +2007 :: 16005 +2008 :: 20595 +2009 :: 24690 +2010 :: 15150 + + */ + + } + + protected void doImport(AcousticImportConfiguration conf, int... years) throws ImportException { + + System.gc(); + + AcousticImportService service = getService(AcousticImportService.class); + + AcousticImportService.POSSIBLE_YEARS.clear(); + + for (int year : years) { + + AcousticImportService.POSSIBLE_YEARS.add(year); + } + + service.doImport(conf, createFakeUser()); + + System.gc(); + } + + protected void fixCellDatas() throws TopiaException, ParseException { + + int nb = 0; + TopiaDAO<Cell> cellDAO = getDAO(Cell.class); + + long nbCells = cellDAO.count(); + for (Cell cell : cellDAO) { + + if (log.isDebugEnabled()) { + log.debug("treat cell " + cell.getName() + " - " + + cell.getCellType().getName()); + } + + if (!cell.isDataEmpty()) { + for (Data data : cell.getData()) { + String name = data.getDataMetadata().getName(); + if (timeStartData.contains(name)) { + fixDateFormat(cell, data); + } else if (positionDatas.contains(name)) { + fixDataPosition(data); + } + } + } + + if (++nb % 10000 == 0) { + if (log.isInfoEnabled()) { + log.info("Flush at " + nb + "/" + nbCells); + } + flushTransaction("Do flush"); + } + } + + commitTransaction("Do commit!"); + } + + private void exportDatabase() throws TopiaException, IOException { + + TopiaDAO<Voyage> voyageDAO = getDAO(Voyage.class); + + Voyage voyage; + if (voyageDAO.count() == 0) { + + // crate voyage + + voyage = voyageDAO.create(Voyage.PROPERTY_MISSION, getEntities(Mission.class).get(0), + Voyage.PROPERTY_NAME, "name", + Voyage.PROPERTY_START_DATE, newDate(), + Voyage.PROPERTY_END_DATE, newDate(), + Voyage.PROPERTY_START_PORT, "ici", + Voyage.PROPERTY_END_PORT, "labas", + Voyage.PROPERTY_DESCRIPTION, "description", + Voyage.PROPERTY_DATUM, "datum" + ); + + commitTransaction("Do commit!"); + } else { + voyage = voyageDAO.findAll().get(0); + } + + commitTransaction("Do commit!"); + + exportDb(voyage, "fixCells"); + + } + + private void fixDateFormat(Cell cell, Data data) throws ParseException { + + String dataValue = data.getDataValue(); + Date oldDate; + try { + oldDate = dateInstance.parse(dataValue); + } catch (ParseException e) { + oldDate = dateInstance2.parse(dataValue); + } + + String newDate = isoDateFormat.format(oldDate); + + if (log.isInfoEnabled()) { + log.info("Treat date " + dataValue + " - to " + newDate); + } + data.setDataValue(newDate); + cell.setName(newDate); + } + + + private void fixDataPosition(Data data) { + + String dataValue = data.getDataValue(); + + double sign = dataValue.endsWith("W") ? -1 : 1; + + double x = Float.valueOf(dataValue.substring(0, dataValue.length() - 1)); + + double x1 = Math.floor(x / 100); + double x2 = Math.floor(x - x1 * 100) / 60; + double x3 = ((x - x1 * 100) - x2 * 60) / 100; + double newValue = sign * (x1 + x2 + x3); + + if (log.isInfoEnabled()) { + log.info("Treat data position " + x + " to " + newValue); + } + + data.setDataValue(String.valueOf(newValue)); + } + +} Property changes on: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/FixCellsIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Added: svn:eol-style + native