Author: fdesbois
Date: 2010-01-19 18:02:46 +0000 (Tue, 19 Jan 2010)
New Revision: 242
Modified:
trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java
trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java
trunk/suiviobsmer-business/src/main/xmi/suiviobsmer.zargo
Log:
Evol #2025 : Some fields in Csv file are "NA" but not reading correctly in ActivityCalendar import
Modified: trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java
===================================================================
--- trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java 2010-01-19 16:20:59 UTC (rev 241)
+++ trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/ImportHelper.java 2010-01-19 18:02:46 UTC (rev 242)
@@ -66,7 +66,7 @@
/**
* CSV headers for Boat
*/
- public static enum BOAT implements ImportHeader {
+ public enum BOAT implements ImportHeader {
/** Boat immatriculation **/
NAVS_COD(20),
/** Boat name **/
@@ -106,7 +106,7 @@
/**
* CSV headers for SamplingPlan
*/
- public static enum SAMPLING implements ImportHeader {
+ public enum SAMPLING implements ImportHeader {
/** SampleRow code **/
PLAN_CODE(6),
/** Company name **/
@@ -160,7 +160,7 @@
/**
* CSV headers for FishingZone
*/
- public static enum FISHING_ZONE implements ImportHeader {
+ public enum FISHING_ZONE implements ImportHeader {
/** FishingZone facade **/
PECHE_FACADE(10),
/** FishingZone sector **/
@@ -188,7 +188,7 @@
/**
* CSV headers for Contact
*/
- public static enum CONTACT implements ImportHeader {
+ public enum CONTACT implements ImportHeader {
/** Contact code (create date time for existing contact) **/
CONT_CODE(0),
/** Contact create date **/
@@ -239,6 +239,55 @@
}
}
+ public enum ACTIVITY_CALENDAR implements ImportHeader {
+ /** ActivityCalendar year **/
+ SYNA_AN,
+ /** ActivityCalendar fiability **/
+ INDQ_COD,
+ /** ActivityMonth month **/
+ SYNA_MOI,
+ /** ActivityMonth harbourCode **/
+ SYNA_POR_COD,
+ /** ActivityMonth nbSeaDays **/
+ SYNA_NOMJDM,
+ /** ActivityMonth nbFishingDays **/
+ SYNA_NOMJDP,
+ /** ActivityMonth nbBoardingPersons **/
+ SYNA_NOMHE,
+ /** ActivityMonth harbourId **/
+ SYNA_TPOR_COD,
+ /** ActivityMonth harbourLibelle **/
+ SYNA_POR_LIB,
+ /** ActivityProfession professionOrder **/
+ META_ORDRE,
+ /** ActivityProfession code **/
+ MET_COD,
+ /** ActivityProfession id **/
+ MET_ID,
+ /** ActivityProfession libelle **/
+ MET_LIB,
+ /** ActivityZone code **/
+ SECT_COD,
+ /** ActivityZone gradiantCode **/
+ GRA_COD,
+ /** ActivityZone gradiantLibelle **/
+ GRA_LIB,
+ /** ActivityZone zoneId **/
+ TSECT_COD,
+ /** ActivityZone zoneLibelle **/
+ SECT_LIB;
+
+ @Override
+ public int forContactCsv() {
+ return -1;
+ }
+
+ @Override
+ public String datePattern() {
+ return "";
+ }
+ }
+
public static int CONTACT_NB_HEADERS = 33;
public static String getHeaderForContactCsv(int index) {
@@ -276,15 +325,6 @@
}
return valid;
}
-//
-// public static Boolean parseContactValidation(String validation) {
-// if (validation.equals("A")) {
-// return Boolean.TRUE;
-// } else if (validation.equals("R")) {
-// return Boolean.FALSE;
-// }
-// return null;
-// }
public static String formatContactMammals(boolean mammals) {
return mammals ? "X" : "";
@@ -346,28 +386,96 @@
return System.currentTimeMillis();
}
+ /**
+ * Read a string value in CsvReader from {@code header} column.
+ * In ActivityCalendarImport the result value can be null if the read value is "NA".
+ *
+ * @param reader CsvReader used to read the value
+ * @param header Column header in the CsvReader
+ * @return the String value read or null if set to "NA" in ActivityCalendar import.
+ * @throws IOException
+ */
public static String read(CsvReader reader, ImportHeader header) throws IOException {
- return reader.get(header.name()).trim();
+ String result = reader.get(header.name()).trim();
+ if (header instanceof ACTIVITY_CALENDAR && result.equals("NA")) {
+ return null;
+ }
+ return result;
}
+ /**
+ * Read an int value in CsvReader from {@code header} column.
+ *
+ * @param reader CsvReader used to read the value
+ * @param header Column header in the CsvReader
+ * @return the int value or -1 if the value read is null
+ * @throws IOException
+ * @see ImportHelper#read(CsvReader, ImportHeader)
+ */
public static int readInt(CsvReader reader, ImportHeader header) throws IOException {
String str = read(reader, header);
+ if (str == null) {
+ return -1;
+ }
return Integer.parseInt(str);
}
+ /**
+ * Read an Integer value in CsvReader from {@code header} column.
+ *
+ * @param reader CsvReader used to read the value
+ * @param header Column header in the CsvReader
+ * @return the Integer value or null if the value read is null
+ * @throws IOException
+ * @see ImportHelper#readInt(CsvReader, ImportHeader)
+ */
+ public static Integer readInteger(CsvReader reader, ImportHeader header) throws IOException {
+ int result = readInt(reader, header);
+ return result != -1 ? result : null;
+ }
+
+ /**
+ * Read two values in CsvReader corresponding to a period from {@code headerBegin} and {@code headEnd} column.
+ *
+ * @param reader CsvReader used to read the value
+ * @param headerBegin Column header in the CsvReader for periodBegin
+ * @param headerEnd Column header in the CsvReader for periodEnd
+ * @return the PeriodDates created from periodBegin and periodEnd read from Csv file
+ * @throws IOException
+ * @throws ParseException
+ * @throws IllegalArgumentException
+ * @see ImportHelper#readDate(CsvReader, ImportHeader)
+ */
public static PeriodDates readPeriod(CsvReader reader, ImportHeader headerBegin, ImportHeader headerEnd)
throws IOException, ParseException, IllegalArgumentException {
Date end = readDate(reader, headerEnd);
Date begin = readDate(reader, headerBegin);
- PeriodDates period = new PeriodDates(begin, end);
- period.initDayOfMonthExtremities();
- return period;
+ if (begin != null && end != null) {
+ PeriodDates period = new PeriodDates(begin, end);
+ period.initDayOfMonthExtremities();
+ return period;
+ }
+ return null;
}
+ /**
+ * Read a Date value in CsvReader from {@code header} column.
+ *
+ * @param reader CsvReader used to read the value
+ * @param header Column header in the CsvReader
+ * @return the Date value or null if the value read is null
+ * @throws IOException
+ * @throws ParseException
+ * @see ImportHelper#read(CsvReader, ImportHeader)
+ */
public static Date readDate(CsvReader reader, ImportHeader header)
throws IOException, ParseException {
DateFormat dateFormat = new SimpleDateFormat(header.datePattern());
- return dateFormat.parse(read(reader, header));
+ String str = read(reader, header);
+ if (str == null) {
+ return null;
+ }
+ return dateFormat.parse(str);
}
}
Modified: trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java
===================================================================
--- trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java 2010-01-19 16:20:59 UTC (rev 241)
+++ trunk/suiviobsmer-business/src/main/java/fr/ifremer/suiviobsmer/impl/ActivityCalendarImport.java 2010-01-19 18:02:46 UTC (rev 242)
@@ -23,6 +23,7 @@
import fr.ifremer.suiviobsmer.*;
import com.csvreader.CsvReader;
+import fr.ifremer.suiviobsmer.ImportHelper.ACTIVITY_CALENDAR;
import fr.ifremer.suiviobsmer.ImportHelper.BOAT;
import fr.ifremer.suiviobsmer.entity.ActivityCalendar;
import fr.ifremer.suiviobsmer.entity.ActivityCalendarDAO;
@@ -161,39 +162,26 @@
while(reader.readRecord()) {
currRow++;
- //long firstTic = System.currentTimeMillis();
int boatImmatriculation = Integer.parseInt(reader.get(BOAT.NAVS_COD.name()).trim());
-// long tic1 = System.currentTimeMillis();
-
- //Boat boat = boatDAO.findByImmatriculation(boatImmatriculation);
-// TopiaContextImplementor topia = (TopiaContextImplementor)transaction;
-// SQLQuery query = topia.getHibernate().createSQLQuery(
-// "SELECT * FROM boat WHERE immatriculation = " + boatImmatriculation);
-//
-// Boat boat = (Boat)query.addEntity(BoatImpl.class).uniqueResult();
- Boat boat = boatDAO.findByImmatriculation(boatImmatriculation); //.add(Boat.IMMATRICULATION, boatImmatriculation).executeToEntity(transaction);
+ Boat boat = boatDAO.findByImmatriculation(boatImmatriculation);
-// long tic2 = System.currentTimeMillis();
-// log.info("findByImmatriculation : " + (tic2 - tic1));
if (boat != null) {
- int year = Integer.parseInt(reader.get("SYNA_AN").trim());
+// int year = Integer.parseInt(reader.get("SYNA_AN").trim());
+ int year = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.SYNA_AN);
+ if (year == -1) {
+ error("Ligne non sauvegardé car l'année (SYNA_AN) n'est pas renseigné", currRow);
+ continue;
+ }
ActivityCalendarKey key = new ActivityCalendarKey(boat, year);
ActivityCalendar calendar = availableCalendars.get(key);
-// tic1 = System.currentTimeMillis();
-
// If not available select it from Database
if (calendar == null) {
-// TopiaContextImplementor topia = (TopiaContextImplementor)transaction;
-// SQLQuery query = topia.getHibernate().createSQLQuery(
-// "SELECT * FROM activityCalendar WHERE boat = '" + key.boatId() + "' AND year = " + key.year());
-//
-// calendar = (ActivityCalendar)query.addEntity(ActivityCalendarImpl.class).uniqueResult();
calendar = calendarDAO.findByProperties(
ActivityCalendar.BOAT, boat,
@@ -201,7 +189,9 @@
// Create it if not exist in Database
if (calendar == null) {
- int fiability = Integer.parseInt(reader.get("INDQ_COD").trim());
+ //int fiability = Integer.parseInt(reader.get("INDQ_COD").trim());
+ // fiability can be -1 if NA is set in CSV file
+ int fiability = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.INDQ_COD);
calendar = calendarDAO.create(
ActivityCalendar.BOAT, key.boat(),
@@ -218,21 +208,17 @@
availableCalendars.put(key, calendar);
}
-// ActivityCalendar calendar = calendarDAO.findByProperties(
-// ActivityCalendar.BOAT, boat,
-// ActivityCalendar.YEAR, year);
+ //int monthNum = Integer.parseInt(reader.get("SYNA_MOI").trim());
+ int monthNum = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.SYNA_MOI);
+ if (monthNum == -1) {
+ error("Ligne non sauvegardé car le mois (SYNA_MOI) n'est pas renseigné", currRow);
+ continue;
+ }
-// tic2 = System.currentTimeMillis();
-// log.info("calendar.findByProperties : " + (tic2 - tic1));
+ //String harbourCode = reader.get("SYNA_POR_COD").trim();
+ String harbourCode = ImportHelper.read(reader, ACTIVITY_CALENDAR.SYNA_POR_COD);
+ boolean active = harbourCode != null && !harbourCode.equals("INA");
-
- String calendarCode = "[ calendrier " + year + ", navire " + boatImmatriculation + " ]";
-
- int monthNum = Integer.parseInt(reader.get("SYNA_MOI").trim());
-
- String harbourCode = reader.get("SYNA_POR_COD").trim();
- boolean active = !harbourCode.equals("INA");
-
ActivityMonth month = calendar.getActivityMonth(monthNum);
if (month == null) {
@@ -242,34 +228,37 @@
month.setActivityProfession(new ArrayList<ActivityProfession>());
-// if (log.isDebugEnabled()) {
-// info("Création du mois " + monthNum + " actif(" + active + ") " + calendarCode, currRow);
-// }
-
calendar.addActivityMonth(month);
}
month.setActive(active);
if (active) {
- String nbSeaDaysString = reader.get("SYNA_NOMJDM").trim();
- Integer nbSeaDays = !nbSeaDaysString.equals("NA") ?
- Integer.valueOf(nbSeaDaysString) : null;
+// String nbSeaDaysString = reader.get("SYNA_NOMJDM").trim();
+// Integer nbSeaDays = !nbSeaDaysString.equals("NA") ?
+// Integer.valueOf(nbSeaDaysString) : null;
+//
+// String nbFishingDaysString = reader.get("SYNA_NOMJDP").trim();
+// Integer nbFishingDays = !nbFishingDaysString.equals("NA") ?
+// Integer.valueOf(nbFishingDaysString) : null;
+//
+// String nbBoardingPersonsString = reader.get("SYNA_NOMHE").trim();
+// Integer nbBoardingPersons = !nbBoardingPersonsString.equals("NA") ?
+// Integer.valueOf(nbBoardingPersonsString) : null;
+//
+// String harbourIdString = reader.get("SYNA_TPOR_COD").trim();
+// int harbourId = !harbourIdString.equals("NA") ? Integer.parseInt(harbourIdString) : -1;
+// String harbourLibelle = reader.get("SYNA_POR_LIB").trim();
+// harbourLibelle = !harbourLibelle.equals("NA") ? harbourLibelle : null;
+// harbourCode = !harbourCode.equals("NA") ? harbourCode : null;
- String nbFishingDaysString = reader.get("SYNA_NOMJDP").trim();
- Integer nbFishingDays = !nbFishingDaysString.equals("NA") ?
- Integer.valueOf(nbFishingDaysString) : null;
+ Integer nbSeaDays = ImportHelper.readInteger(reader, ACTIVITY_CALENDAR.SYNA_NOMJDM);
+ Integer nbFishingDays = ImportHelper.readInteger(reader, ACTIVITY_CALENDAR.SYNA_NOMJDP);
+ Integer nbBoardingPersons = ImportHelper.readInteger(reader, ACTIVITY_CALENDAR.SYNA_NOMHE);
+
+ int harbourId = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.SYNA_TPOR_COD);
+ String harbourLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.SYNA_POR_LIB);
- String nbBoardingPersonsString = reader.get("SYNA_NOMHE").trim();
- Integer nbBoardingPersons = !nbBoardingPersonsString.equals("NA") ?
- Integer.valueOf(nbBoardingPersonsString) : null;
-
- String harbourIdString = reader.get("SYNA_TPOR_COD").trim();
- int harbourId = !harbourIdString.equals("NA") ? Integer.parseInt(harbourIdString) : -1;
- String harbourLibelle = reader.get("SYNA_POR_LIB").trim();
- harbourLibelle = !harbourLibelle.equals("NA") ? harbourLibelle : null;
- harbourCode = !harbourCode.equals("NA") ? harbourCode : null;
-
month.setHarbourId(harbourId);
month.setHarbourCode(harbourCode);
month.setHarbourLibelle(harbourLibelle);
@@ -277,10 +266,17 @@
month.setNbFishingDays(nbFishingDays);
month.setNbSeaDays(nbSeaDays);
- int professionOrder = Integer.parseInt(reader.get("META_ORDRE").trim());
- String professionCode = reader.get("MET_COD").trim();
- int professionId = Integer.parseInt(reader.get("MET_ID").trim());
- String professionLibelle = reader.get("MET_LIB").trim();
+// int professionOrder = Integer.parseInt(reader.get("META_ORDRE").trim()); // NA
+// String professionCode = reader.get("MET_COD").trim();
+// int professionId = Integer.parseInt(reader.get("MET_ID").trim());
+// String professionLibelle = reader.get("MET_LIB").trim();
+
+ // Can be equals to -1 if set to NA in Csv File (not a problem for ordering Profession)
+ int professionOrder = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.META_ORDRE);
+ String professionCode = ImportHelper.read(reader, ACTIVITY_CALENDAR.MET_COD);
+ int professionId = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.MET_ID);
+ String professionLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.MET_LIB);
+
ActivityProfession profession = month.getActivityProfession(professionOrder);
if (profession == null) {
@@ -291,12 +287,6 @@
profession.setActivityZone(new ArrayList<ActivityZone>());
-// if (log.isDebugEnabled()) {
-// info("Création du métier " + professionOrder + " code(" + professionCode + ") " +
-// "[ mois " + monthNum + " ] " +
-// calendarCode, currRow);
-// }
-
month.addActivityProfession(profession);
}
@@ -304,12 +294,18 @@
profession.setCode(professionCode);
profession.setLibelle(professionLibelle);
- String zoneCode = reader.get("SECT_COD").trim();
- int gradiantCode = Integer.parseInt(reader.get("GRA_COD").trim());
- String gradiantLibelle = reader.get("GRA_LIB").trim();
- int zoneId = Integer.parseInt(reader.get("TSECT_COD").trim());
- String zoneLibelle = reader.get("SECT_LIB").trim();
+// String zoneCode = reader.get("SECT_COD").trim();
+// int gradiantCode = Integer.parseInt(reader.get("GRA_COD").trim());
+// String gradiantLibelle = reader.get("GRA_LIB").trim();
+// int zoneId = Integer.parseInt(reader.get("TSECT_COD").trim());
+// String zoneLibelle = reader.get("SECT_LIB").trim();
+ String zoneCode = ImportHelper.read(reader, ACTIVITY_CALENDAR.SECT_COD);
+ int gradiantCode = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.GRA_COD);
+ String gradiantLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.GRA_LIB);
+ int zoneId = ImportHelper.readInt(reader, ACTIVITY_CALENDAR.TSECT_COD);
+ String zoneLibelle = ImportHelper.read(reader, ACTIVITY_CALENDAR.SECT_LIB);
+
ActivityZone zone = profession.getActivityZone(zoneCode);
if (zone == null) {
@@ -317,19 +313,7 @@
ActivityZone.ZONE_CODE, zoneCode,
ActivityZone.ACTIVITY_PROFESSION, profession);
-// if (log.isDebugEnabled()) {
-// info("Création de la zone " + zoneCode + " [ mois " + monthNum + " ] " +
-// calendarCode, currRow);
-// }
-
profession.addActivityZone(zone);
-
- } else {
-// if (log.isDebugEnabled()) {
-// info("Ecrasement de la zone " + zoneCode + " [ mois " + monthNum + " ] " +
-// calendarCode, currRow);
-// }
- //warn("Zone déjà existante [" + zoneCode + "] ! :: " + Arrays.toString(reader.getValues()), currRow);
}
zone.setZoneId(zoneId);
@@ -344,11 +328,6 @@
warn("Navire inexistant dans le référentiel : " + boatImmatriculation, currRow);
}
-// if (log.isDebugEnabled()) {
-// long tic = System.currentTimeMillis();
-// log.debug("Time : " + DurationFormatUtils.formatDurationHMS(tic - firstTic));
-// }
-
if (result % 1000 == 0) {
tic1 = System.currentTimeMillis();
@@ -360,16 +339,6 @@
log.debug("commit : " + (tic2 - tic1));
}
firstTic = ImportHelper.logTimeAndMemory(log, firstTic, "calendrier d'activité ligne " + currRow);
-// if (log.isInfoEnabled()) {
-// log.info("RUNNING... Import calendrier d'activité ligne " + currRow);
-// Runtime runtime = Runtime.getRuntime();
-// long mem = runtime.totalMemory() - runtime.freeMemory();
-// long memMega = mem / 1024 / 1024;
-// log.info("Memory : " + memMega + " Mo");
-// long tic = System.currentTimeMillis();
-// log.info("Time : " + DurationFormatUtils.formatDurationHMS(tic - firstTic));
-// firstTic = System.currentTimeMillis();
-// }
}
}
@@ -378,16 +347,9 @@
long stopTime = System.currentTimeMillis();
- Runtime runtime = Runtime.getRuntime();
+ // Force garbage collector
+ System.gc();
- long mem = (runtime.totalMemory() - runtime.freeMemory()) / 1048576;
- log.info("Memory before gc : " + mem + " Mo");
-
- System.gc();
-
- mem = (runtime.totalMemory() - runtime.freeMemory()) / 1048576;
- log.info("Memory after gc : " + mem + " Mo");
-
String execTime = DurationFormatUtils.formatDurationHMS(stopTime - startTime);
info("Nombre de ligne ajouté : " + result, -1);
Modified: trunk/suiviobsmer-business/src/main/xmi/suiviobsmer.zargo
===================================================================
(Binary files differ)
