Author: tchemit Date: 2011-12-19 22:45:05 +0100 (Mon, 19 Dec 2011) New Revision: 205 Url: http://forge.codelutin.com/repositories/revision/echobase/205 Log: improve DbImportService Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java trunk/echobase-ui/src/main/env/dev/resources/log4j.properties trunk/echobase-ui/src/main/env/prod/resources/log4j.properties Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java 2011-12-19 19:18:47 UTC (rev 204) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java 2011-12-19 21:45:05 UTC (rev 205) @@ -175,6 +175,8 @@ result.add(EchoBaseEntityEnum.OperationMetadata); result.add(EchoBaseEntityEnum.GearMetadata); result.add(EchoBaseEntityEnum.Gear); + result.add(EchoBaseEntityEnum.LengthAgeKey); + result.add(EchoBaseEntityEnum.LengthWeightKey); return result.toArray(new EchoBaseEntityEnum[result.size()]); } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java 2011-12-19 19:18:47 UTC (rev 204) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/DbImportService.java 2011-12-19 21:45:05 UTC (rev 205) @@ -38,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.TopiaException; +import org.nuiton.topia.framework.TopiaContextImplementor; import org.nuiton.topia.persistence.TopiaDAO; import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.util.StringUtil; @@ -99,10 +100,6 @@ log.warn("There is " + missingEntries.size() + "missing mandatory files " + " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false)); } - -// throw new EchoBaseTechnicalException( -// "There is some missing mandatory files " + -// missingEntries + " in import " + file); } return result; @@ -187,125 +184,120 @@ } - public <E extends TopiaEntity> void importFile(MetaFilenameAware entry, - Reader reader, - CsvImportResult csvResult) throws IOException { + public void importFile(MetaFilenameAware entry, + Reader reader, + CsvImportResult csvResult) throws IOException, TopiaException { - DbEditorService service = getDbEditorService(); - long s0; + long s0 = TimeLog.getTime(); - s0 = TimeLog.getTime(); + if (log.isInfoEnabled()) { + log.info("Will import " + entry); + } if (entry instanceof AssociationMeta) { // load a association input + importAssociationfile((AssociationMeta) entry, reader, csvResult); - AssociationMeta meta = (AssociationMeta) entry; + } else { - if (log.isInfoEnabled()) { - log.info("Will import " + meta); - } + // normal entity table import + importEntityFile((TableMeta) entry, reader, csvResult); + } - EchoBaseEntityEnum source = meta.getSource(); - EchoBaseEntityEnum target = meta.getTarget(); - timeLog.log(s0, "importFile::getTargetEntities"); -// List<TopiaEntity> targetEntities = getEntities(target); -// Map<String, TopiaEntity> targetsById = -// Maps.uniqueIndex(targetEntities, EchoBaseFunctions.TO_TOPIAID); + timeLog.log(s0, "importFile::done"); - ImportModel<Map<String, Object>> model = - EntityAssociationImportModel.newImportModel( - getConfiguration().getCsvSeparator(), - meta - ); + s0 = TimeLog.getTime(); -// EchoBaseEntityEnum sourceType = meta.getSource(); + flushTransaction(); - ImportToMap importer = MyImportToMap.newImport(model, reader); + timeLog.log(s0, "importFile::flushTransaction"); + } - s0 = TimeLog.getTime(); -// List<E> sourceEntities = getEntities(sourceType); - timeLog.log(s0, "importFile::getSourceEntities"); -// Map<String, E> sourcesById = -// Maps.uniqueIndex(sourceEntities, EchoBaseFunctions.TO_TOPIAID); - StringBuilder builder = new StringBuilder(); - try { - s0 = TimeLog.getTime(); - String updateString = "UPDATE " + target.getContract().getSimpleName() + " SET " + source.getContract().getSimpleName() + " = '%s' WHERE topiaId ='%s';"; - if (log.isInfoEnabled()) { - log.info("Will apply " + updateString); - } - int compt = 0; - for (Map<String, Object> row : importer) { - String topiaId = (String) row.get(TopiaEntity.TOPIA_ID); - String[] associations = (String[]) row.get("target"); - for (String association : associations) { - builder.append(String.format(updateString, topiaId, association)).append('\n'); - compt++; - if (compt == 10000) { - // flush it - getTransaction().executeSQL(builder.toString()); - builder = new StringBuilder(); - } - } - csvResult.incrementsNumberUpdated(); - } - timeLog.log(s0, "importFile::import"); - } catch (TopiaException eee) { - throw new EchoBaseTechnicalException( - "Could not execute sql script", eee); - } finally { + public MetaFilenameAware[] getEntries() { - importer.close(); - } + List<MetaFilenameAware> entities = Lists.newArrayList(); + List<MetaFilenameAware> associations = Lists.newArrayList(); - } else { + addEntries(entities, associations, EntitiesUtil.getReferenceTypesForCopy()); + addEntries(entities, associations, EntitiesUtil.getDataTypesforCopy()); + entities.addAll(associations); + return entities.toArray(new MetaFilenameAware[entities.size()]); + } - // normal entity table import + protected void importEntityFile(TableMeta meta, Reader reader, CsvImportResult csvResult) { - TableMeta meta = (TableMeta) entry; + ImportModel<TopiaEntity> model = getDbEditorService().buildForImport(meta); - if (log.isInfoEnabled()) { - log.info("Will import " + meta); - } + Import<TopiaEntity> importer = Import.newImport(model, reader); - ImportModel<TopiaEntity> model = service.buildForImport(meta); + try { + TopiaDAO<TopiaEntity> dao = getDAO(meta.getSource()); + for (TopiaEntity entity : importer) { - Import<TopiaEntity> importer = Import.newImport(model, reader); + TopiaEntity entityToSave = dao.create(TopiaEntity.TOPIA_ID, + entity.getTopiaId()); - try { - TopiaDAO<TopiaEntity> dao = getDAO(meta.getSource()); - s0 = TimeLog.getTime(); - for (TopiaEntity entity : importer) { + meta.copy(entity, entityToSave); - TopiaEntity entityToSave = dao.create(TopiaEntity.TOPIA_ID, - entity.getTopiaId()); + csvResult.incrementsNumberUpdated(); + } - meta.copy(entity, entityToSave); + } catch (Exception eee) { + throw new EchoBaseTechnicalException("Could not import entities of type " + meta.getSource(), eee); + } finally { - csvResult.incrementsNumberUpdated(); - } - timeLog.log(s0, "importFile::import"); - } catch (Exception eee) { - throw new EchoBaseTechnicalException("Could not import entities of type " + meta.getSource(), eee); - } finally { - - importer.close(); - } + importer.close(); } } - public MetaFilenameAware[] getEntries() { + protected void flushTransaction() throws TopiaException { + TopiaContextImplementor tx = (TopiaContextImplementor) getTransaction(); + tx.getHibernate().flush(); + } - List<MetaFilenameAware> entities = Lists.newArrayList(); - List<MetaFilenameAware> associations = Lists.newArrayList(); + protected void importAssociationfile(AssociationMeta meta, Reader reader, CsvImportResult csvResult) { + EchoBaseEntityEnum source = meta.getSource(); + EchoBaseEntityEnum target = meta.getTarget(); - addEntries(entities, associations, EntitiesUtil.getReferenceTypesForCopy()); - addEntries(entities, associations, EntitiesUtil.getDataTypesforCopy()); - entities.addAll(associations); - return entities.toArray(new MetaFilenameAware[entities.size()]); + ImportModel<Map<String, Object>> model = + EntityAssociationImportModel.newImportModel( + getConfiguration().getCsvSeparator(), + meta + ); + + ImportToMap importer = MyImportToMap.newImport(model, reader); + + StringBuilder builder = new StringBuilder(); + try { + String updateString = "UPDATE " + target.getContract().getSimpleName() + " SET " + source.getContract().getSimpleName() + " = '%s' WHERE topiaId ='%s';"; + if (log.isDebugEnabled()) { + log.debug("Will apply " + updateString); + } + int compt = 0; + for (Map<String, Object> row : importer) { + String topiaId = (String) row.get(TopiaEntity.TOPIA_ID); + String[] associations = (String[]) row.get("target"); + for (String association : associations) { + builder.append(String.format(updateString, topiaId, association)).append('\n'); + compt++; + if (compt == 10000) { + // flush it + getTransaction().executeSQL(builder.toString()); + builder = new StringBuilder(); + } + } + csvResult.incrementsNumberUpdated(); + } + } catch (TopiaException eee) { + throw new EchoBaseTechnicalException( + "Could not execute sql script", eee); + } finally { + + importer.close(); + } } protected TableMeta getTableMeta(EchoBaseEntityEnum tableName) { Modified: trunk/echobase-ui/src/main/env/dev/resources/log4j.properties =================================================================== --- trunk/echobase-ui/src/main/env/dev/resources/log4j.properties 2011-12-19 19:18:47 UTC (rev 204) +++ trunk/echobase-ui/src/main/env/dev/resources/log4j.properties 2011-12-19 21:45:05 UTC (rev 205) @@ -34,3 +34,4 @@ log4j.logger.org.nuiton=WARN log4j.logger.org.nuiton.topia.migration=INFO log4j.logger.com.opensymphony.xwork2.util.logging.commons.CommonsLogger=ERROR +log4j.logger.org.apache.struts2.util.TokenHelper=ERROR Modified: trunk/echobase-ui/src/main/env/prod/resources/log4j.properties =================================================================== --- trunk/echobase-ui/src/main/env/prod/resources/log4j.properties 2011-12-19 19:18:47 UTC (rev 204) +++ trunk/echobase-ui/src/main/env/prod/resources/log4j.properties 2011-12-19 21:45:05 UTC (rev 205) @@ -32,3 +32,4 @@ log4j.logger.fr.ifremer.echobase=INFO log4j.logger.org.nuiton.topia.migration=INFO log4j.logger.com.opensymphony.xwork2.util.logging.commons.CommonsLogger=ERROR +log4j.logger.org.apache.struts2.util.TokenHelper=ERROR