This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository observe. See http://git.codelutin.com/observe.git commit 39fb53da85bd0305d59054c742d33e7d76bfa5e8 Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 27 10:25:33 2015 +0200 La création d'une base est effective avec les imports au démarrage (See #7482) --- .../service/DataSourceDumpProducerService.java | 7 +- .../dto/DataSourceCreateConfigurationDto.java | 7 +- .../ird/observe/services/ObserveServiceTopia.java | 167 ++++++++++++++++++++- .../DataSourceDumpProducerServiceTopia.java | 160 +++----------------- .../services/service/DataSourceServiceTopia.java | 67 +++++++-- .../services/ApplicationContextResource.java | 50 ++++++ .../ird/observe/services/DataSourceResource.java | 24 +-- .../DataSourceDumpProducerServiceTopiaTest.java | 13 +- .../service/DataSourceServiceTopiaTest.java | 58 +++---- 9 files changed, 342 insertions(+), 211 deletions(-) diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java index 816dd3d..911f6ef 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java @@ -1,5 +1,6 @@ package fr.ird.observe.services.service; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.services.ObserveService; /** @@ -7,10 +8,12 @@ import fr.ird.observe.services.ObserveService; * * @author Tony Chemit - chemit@codelutin.com */ -public interface DataSourceDumpProducerService extends ObserveService{ +public interface DataSourceDumpProducerService extends ObserveService { byte[] getReferentialDump(); - byte[] getDataDump(String... importDataIds); + byte[] getAllDataDump(); + + byte[] getDataDump(ImmutableSet<String> importDataIds); } diff --git a/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java b/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java index ec129f9..381a974 100644 --- a/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java +++ b/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java @@ -1,6 +1,7 @@ package fr.ird.observe.services.dto; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.services.configuration.ObserveDataSourceConfiguration; /** @@ -30,7 +31,7 @@ public class DataSourceCreateConfigurationDto extends AbstractObserveDto { /** * Les données à importer (ids des marées) (Optionel). */ - protected String[] optionalImportDataIds; + protected ImmutableSet<String> optionalImportDataIds; /** * Pour autoriser la création d'une base vide (utilisé pour les bases temporaires) @@ -69,7 +70,7 @@ public class DataSourceCreateConfigurationDto extends AbstractObserveDto { return optionalImportDataDataSourceConfiguration; } - public String[] getImportDataIds() { + public ImmutableSet<String> getImportDataIds() { return optionalImportDataIds; } @@ -83,7 +84,7 @@ public class DataSourceCreateConfigurationDto extends AbstractObserveDto { this.optionalImportReferentialDataSourceConfiguration = importReferentialDataSourceConfiguration; } - public void setImportDataConfiguration(ObserveDataSourceConfiguration importDataDataSourceConfiguration, String... importDataIds) { + public void setImportDataConfiguration(ObserveDataSourceConfiguration importDataDataSourceConfiguration, ImmutableSet<String> importDataIds) { Preconditions.checkNotNull(importDataDataSourceConfiguration, "'importDataDataSourceConfiguration' can't be null."); Preconditions.checkNotNull(importDataIds, "'importDataIds' can't be null."); this.optionalImportDataDataSourceConfiguration = importDataDataSourceConfiguration; diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java index 263af71..8c2f811 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java @@ -4,17 +4,25 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.ObserveTopiaPersistenceContext; +import fr.ird.observe.entities.Entities; import fr.ird.observe.entities.constants.ReferenceLocale; +import fr.ird.observe.entities.longline.TripLongline; import fr.ird.observe.entities.referentiel.ReferenceEntity; import fr.ird.observe.services.builder.DtoToEntityBuilder; import fr.ird.observe.services.builder.EntityToDtoBuilder; import fr.ird.observe.services.builder.EntityToReferenceDtoBuilder; import fr.ird.observe.services.builder.EntityToReferentialReferenceDtoBuilder; +import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; +import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; +import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; import fr.ird.observe.services.dto.FormDto; import fr.ird.observe.services.dto.FormDtos; import fr.ird.observe.services.dto.IdDto; +import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.services.dto.ObserveDtosInitializer; import fr.ird.observe.services.dto.ObserveModelInitializerRunner; import fr.ird.observe.services.dto.ReferenceDto; @@ -23,10 +31,17 @@ import fr.ird.observe.services.dto.ReferenceSetDtos; import fr.ird.observe.services.dto.constants.ReferentialLocale; import fr.ird.observe.services.dto.referential.ReferentialDto; import fr.ird.observe.services.service.DataNotFoundException; +import fr.ird.observe.services.service.DataSourceService; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.topia.persistence.TopiaDao; import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.topia.replication.TopiaReplicationService; +import org.nuiton.topia.replication.model.ReplicationModel; +import org.nuiton.util.StringUtil; +import java.io.File; import java.util.Date; import java.util.LinkedHashSet; import java.util.List; @@ -41,6 +56,9 @@ import java.util.Set; */ public class ObserveServiceTopia implements ObserveService { + /** Logger. */ + private static final Log log = LogFactory.getLog(ObserveServiceTopia.class); + protected ObserveServiceContextTopia serviceContext; protected static ImmutableMap<Class<?>, Class<?>> DTO_TO_ENTITY_TYPES; @@ -220,13 +238,13 @@ public class ObserveServiceTopia implements ObserveService { } - protected <E extends TopiaEntity, D extends IdDto> ReferenceDto<D> entityToReferenceDto(Class<D> dtoType, E entity) { + protected <E extends TopiaEntity, D extends IdDto> ReferenceDto<D> entityToReferenceDto(Class<D> dtoType, E entity) { ReferenceDto<D> referenceDto; if (entity instanceof ReferenceEntity) { - referenceDto = EntityToReferentialReferenceDtoBuilder.build((Class)dtoType, serviceContext.getReferentialLocale(), entity); + referenceDto = EntityToReferentialReferenceDtoBuilder.build((Class) dtoType, serviceContext.getReferentialLocale(), entity); } else { @@ -260,4 +278,149 @@ public class ObserveServiceTopia implements ObserveService { public ObserveTopiaPersistenceContext getTopiaPersistenceContext() { return serviceContext.getTopiaPersistenceContext(); } + + protected ObserveDataSourceConfigurationTopiaH2 createTemporaryDatabaseFromDump(String prefix, byte... importContent) { + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = new ObserveDataSourceConfigurationTopiaH2(); + temporaryDataSourceConfiguration.setDbName("obstuna"); + temporaryDataSourceConfiguration.setLabel(prefix); + temporaryDataSourceConfiguration.setUsername("sa"); + temporaryDataSourceConfiguration.setPassword('s', 'a'); + File temporaryDataSourceDirectory = serviceContext.createTemporaryDirectory(prefix); + temporaryDataSourceConfiguration.setDirectory(temporaryDataSourceDirectory); + + DataSourceService dataSourceService = serviceContext.newService(temporaryDataSourceConfiguration, DataSourceService.class); + + try { + DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); + dataSourceCreateConfiguration.setImportDatabase(importContent); + dataSourceService.create(dataSourceCreateConfiguration); + } catch (IncompatibleDataSourceCreateConfigurationException | DataSourceCreateWithNoReferentialImportException e) { + //FIXME Avoir une exception plus concrete + throw new RuntimeException("Could not create temporary database: " + prefix, e); + } + + return temporaryDataSourceConfiguration; + } + protected ObserveDataSourceConfigurationTopiaH2 createTemporaryDatabase(String prefix) { + + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = new ObserveDataSourceConfigurationTopiaH2(); + temporaryDataSourceConfiguration.setDbName("obstuna"); + temporaryDataSourceConfiguration.setLabel(prefix); + temporaryDataSourceConfiguration.setUsername("sa"); + temporaryDataSourceConfiguration.setPassword('s', 'a'); + File temporaryDataSourceDirectory = serviceContext.createTemporaryDirectory(prefix); + temporaryDataSourceConfiguration.setDirectory(temporaryDataSourceDirectory); + + DataSourceService dataSourceService = serviceContext.newService(temporaryDataSourceConfiguration, DataSourceService.class); + + try { + DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); + dataSourceCreateConfiguration.setCanCreateEmptyDatabase(true); + dataSourceService.create(dataSourceCreateConfiguration); + } catch (IncompatibleDataSourceCreateConfigurationException | DataSourceCreateWithNoReferentialImportException e) { + //FIXME Avoir une exception plus concrete + throw new RuntimeException("Could not create temporary database: " + prefix, e); + } + + return temporaryDataSourceConfiguration; + + } + + protected void replicateReferential(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { + + long t0 = System.nanoTime(); + + TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); + ReplicationModel model = service.prepareForAll(Entities.REFERENCE_ENTITIES); + + try { + service.doReplicate(model, temporaryTopiaApplicationContext); + } catch (Exception e) { + //TODO Avoir une exception concrete + throw new RuntimeException("Could not replicate referantial", e); + } + + + if (log.isInfoEnabled()) { + log.info(String.format("Referential replication done in %s", StringUtil.convertTime(t0, System.nanoTime()))); + } + + } + + protected void replicateData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, + ObserveTopiaApplicationContext temporaryTopiaApplicationContext, + Set<String> ids) { + try { + + + for (String id : ids) { + long t1 = System.nanoTime(); + + replicateOneData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, id); + + if (log.isInfoEnabled()) { + log.info(String.format("Data replication [%s] done in %s", id, StringUtil.convertTime(t1, System.nanoTime()))); + } + } + + } catch (Exception e) { + throw new RuntimeException("Could not replicate data", e); + } + } + + /** + * Duplication de l'unique donnée observateur depuis ce service vers le service donné. + * + * La duplication utilise une transaction dédiée afin de ne pas saturer le serveur + * et aussi une meilleure maitrise du rollback en cas d'une erreur. + * + * Voir http://forge.codelutin.com/issues/4837 + * + * @param id l'id de la donnée à répliquer + */ + protected void replicateOneData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext, String id) { + + TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); + + ReplicationModel model = service.prepare(Entities.DATA_ENTITIES, false, id); + + // To fix missing tdr associations (see https://forge.codelutin.com/issues/6611) + Multimap<String, String> tdrAssociation = null; + + try (ObserveTopiaPersistenceContext sourcePersistenceContext = sourceTopiaApplicationContext.newPersistenceContext()) { + + TopiaEntity e = sourcePersistenceContext.findByTopiaId(id); + + if (e instanceof TripLongline) { + + // Grab tdr missing associations (see https://forge.codelutin.com/issues/6611) + if (log.isInfoEnabled()) { + log.info("Should keep SetLongline - Tdr association ids for: " + id); + } + + tdrAssociation = sourcePersistenceContext.getTdrDao().getTdrIdsBySeineIds((TripLongline) e); + + } + + } + + // do the replicate + try { + service.doReplicate(model, temporaryTopiaApplicationContext); + } catch (Exception e) { + throw new RuntimeException("Could not replicate data: " + id, e); + } + + if (tdrAssociation != null) { + + // Apply back tdr missing associations (see https://forge.codelutin.com/issues/6611) + try (ObserveTopiaPersistenceContext temporaryPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { + + temporaryPersistenceContext.getTdrDao().applyTdrAssociationFix(tdrAssociation); + + } + + } + } + } diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java index 536e8e0..203099d 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java @@ -1,31 +1,19 @@ package fr.ird.observe.services.service; -import com.google.common.collect.Multimap; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.ObserveTopiaPersistenceContext; -import fr.ird.observe.entities.Entities; -import fr.ird.observe.entities.longline.TripLongline; import fr.ird.observe.services.ObserveServiceTopia; import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; -import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; -import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; -import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.topia.replication.TopiaReplicationService; -import org.nuiton.topia.replication.model.ReplicationModel; import org.nuiton.util.StringUtil; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; /** * Created on 23/08/15. @@ -37,7 +25,7 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl /** Logger. */ private static final Log log = LogFactory.getLog(DataSourceDumpProducerServiceTopia.class); - public static final String EXPORT_SQL_STATEMENT = "SCRIPT NOPASSWORDS NOSETTINGS TO '%s' CHARSET 'UTF-8' SCHEMA OBSERVE_COMMON, OBSERVE_SEINE, OBSERVE_LONGLINE;"; + protected static final String EXPORT_SQL_STATEMENT = "SCRIPT NOPASSWORDS NOSETTINGS TO '%s' COMPRESSION GZIP CHARSET 'UTF-8' SCHEMA OBSERVE_COMMON, OBSERVE_SEINE, OBSERVE_LONGLINE, PUBLIC;"; @Override public byte[] getReferentialDump() { @@ -62,43 +50,23 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } - protected void replicateReferential(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { - - long t0 = System.nanoTime(); - - TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); - ReplicationModel model = service.prepareForAll(Entities.REFERENCE_ENTITIES); - - try { - service.doReplicate(model, temporaryTopiaApplicationContext); - } catch (Exception e) { - //TODO Avoir une exception concrete - throw new RuntimeException("Could not replicate referantial", e); - } + @Override + public byte[] getAllDataDump() { + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + List<String> tripSeineIds = getTopiaPersistenceContext().getTripSeineDao().findAllIds(); + builder.addAll(tripSeineIds); + List<String> tripLonglineIds = getTopiaPersistenceContext().getTripLonglineDao().findAllIds(); + builder.addAll(tripLonglineIds); - if (log.isInfoEnabled()) { - log.info(String.format("Referential replication done in %s", StringUtil.convertTime(t0, System.nanoTime()))); - } + ImmutableSet<String> ids = builder.build(); + byte[] content = getDataDump(ids); + return content; } @Override - public byte[] getDataDump(String... importDataIds) { - - Set<String> ids = new LinkedHashSet<>(); - - if (importDataIds.length == 0) { - - // cas limite où on exporte toutes les données - List<String> tripSeineIds = getTopiaPersistenceContext().getTripSeineDao().findAllIds(); - ids.addAll(tripSeineIds); - List<String> tripLonglineIds = getTopiaPersistenceContext().getTripLonglineDao().findAllIds(); - ids.addAll(tripLonglineIds); - } else { - - ids.addAll(Arrays.asList(importDataIds)); - } + public byte[] getDataDump(ImmutableSet<String> importDataIds) { long t0 = System.nanoTime(); @@ -113,26 +81,12 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl replicateReferential(sourceTopiaApplicationContext, temporaryTopiaApplicationContext); - try { - - t0 = System.nanoTime(); - - for (String id : ids) { - long t1 = System.nanoTime(); - - replicateOneData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, id); - - if (log.isInfoEnabled()) { - log.info(String.format("Data replication [%s] done in %s", id, StringUtil.convertTime(t1, System.nanoTime()))); - } - } + t0 = System.nanoTime(); - if (log.isInfoEnabled()) { - log.info(String.format("Data replication for %s data done in %s", ids.size(), StringUtil.convertTime(t0, System.nanoTime()))); - } + replicateData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, importDataIds); - } catch (Exception e) { - throw new RuntimeException("Could not replicate data", e); + if (log.isInfoEnabled()) { + log.info(String.format("Data replication for %s data done in %s", importDataIds.size(), StringUtil.convertTime(t0, System.nanoTime()))); } Path dumpFile = exportToFile(temporaryDataSourceConfiguration, temporaryTopiaApplicationContext); @@ -142,31 +96,6 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } - protected ObserveDataSourceConfigurationTopiaH2 createTemporaryDatabase(String prefix) { - - ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = new ObserveDataSourceConfigurationTopiaH2(); - temporaryDataSourceConfiguration.setDbName("obstuna"); - temporaryDataSourceConfiguration.setLabel(prefix); - temporaryDataSourceConfiguration.setUsername("sa"); - temporaryDataSourceConfiguration.setPassword('s', 'a'); - File temporaryDataSourceDirectory = serviceContext.createTemporaryDirectory(prefix); - temporaryDataSourceConfiguration.setDirectory(temporaryDataSourceDirectory); - - DataSourceService dataSourceService = serviceContext.newService(temporaryDataSourceConfiguration, DataSourceService.class); - - try { - DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); - dataSourceCreateConfiguration.setCanCreateEmptyDatabase(true); - dataSourceService.create(dataSourceCreateConfiguration); - } catch (IncompatibleDataSourceCreateConfigurationException | DataSourceCreateWithNoReferentialImportException e) { - //FIXME Avoir une exception plus concrete - throw new RuntimeException("Could not create temporary database: " + prefix, e); - } - - return temporaryDataSourceConfiguration; - - } - protected Path exportToFile(ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { @@ -196,59 +125,4 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } } - /** - * Duplication de l'unique donnée observateur depuis ce service vers le service donné. - * - * La duplication utilise une transaction dédiée afin de ne pas saturer le serveur - * et aussi une meilleure maitrise du rollback en cas d'une erreur. - * - * Voir http://forge.codelutin.com/issues/4837 - * - * @param id l'id de la donnée à répliquer - */ - protected void replicateOneData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext, String id) { - - TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); - - ReplicationModel model = service.prepare(Entities.DATA_ENTITIES, false, id); - - // To fix missing tdr associations (see https://forge.codelutin.com/issues/6611) - Multimap<String, String> tdrAssociation = null; - - try (ObserveTopiaPersistenceContext sourcePersistenceContext = sourceTopiaApplicationContext.newPersistenceContext()) { - - TopiaEntity e = getTopiaPersistenceContext().findByTopiaId(id); - - if (e instanceof TripLongline) { - - // Grab tdr missing associations (see https://forge.codelutin.com/issues/6611) - if (log.isInfoEnabled()) { - log.info("Should keep SetLongline - Tdr association ids for: " + id); - } - - tdrAssociation = sourcePersistenceContext.getTdrDao().getTdrIdsBySeineIds((TripLongline) e); - - } - - } - - // do the replicate - try { - service.doReplicate(model, temporaryTopiaApplicationContext); - } catch (Exception e) { - throw new RuntimeException("Could not replicate data: " + id, e); - } - - if (tdrAssociation != null) { - - // Apply back tdr missing associations (see https://forge.codelutin.com/issues/6611) - try (ObserveTopiaPersistenceContext observeTopiaPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { - - observeTopiaPersistenceContext.getTdrDao().applyTdrAssociationFix(tdrAssociation); - - } - - } - } - } diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java index 52ea0f3..849d92b 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java @@ -1,10 +1,12 @@ package fr.ird.observe.services.service; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.services.ObserveServiceTopia; import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfiguration; +import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaSupport; import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; @@ -61,36 +63,79 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS } else { - if (dataSourceCreateConfiguration.isImportReferential()) { + boolean importReferential = dataSourceCreateConfiguration.isImportReferential(); + boolean importData = dataSourceCreateConfiguration.isImportData(); + + boolean importStandaloneReferantial = importReferential; + if (importReferential && importData) { + + // Si on est sur la même source de données pour les deux imports, on fait tout sur la même base temporaire + + ObserveDataSourceConfiguration importReferentialDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); + ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); + + importStandaloneReferantial = !importReferentialDataSourceConfiguration.equals(importDataSourceConfiguration); + + } + + boolean referantialImported = false; + + if (importStandaloneReferantial) { if (log.isInfoEnabled()) { - log.info("Import referentials."); + log.info("Import referential."); } ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); + // Récupération du dump qui contient le référentiel DataSourceDumpProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, DataSourceDumpProducerService.class); - byte[] referentialDump = dumpProducerService.getReferentialDump(); - executeGzipSqlStatements(topiaApplicationContext, "ImportReferential_" + importDataSourceConfiguration.getLabel(), referentialDump); + // création d'une base temporaire pour y importer le référentiel + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = createTemporaryDatabaseFromDump("ImportReferential",referentialDump ); + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); +// executeGzipSqlStatements(temporaryTopiaApplicationContext, "ImportReferential_" + importDataSourceConfiguration.getLabel(), referentialDump); + + // Réplication du référentiel + replicateReferential(temporaryTopiaApplicationContext, topiaApplicationContext); + + referantialImported = true; } - if (dataSourceCreateConfiguration.isImportData()) { + + if (importData) { ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); - String[] importDataIds = dataSourceCreateConfiguration.getImportDataIds(); + ImmutableSet<String> importDataIds = dataSourceCreateConfiguration.getImportDataIds(); + + // Récupération du dump qui contient le référentiel et les données + DataSourceDumpProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, DataSourceDumpProducerService.class); + byte[] dataDump = dumpProducerService.getDataDump(importDataIds); + + // création d'une base temporaire pour y importer le référentiel et les données + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = createTemporaryDatabaseFromDump("ImportData",dataDump); + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); +// executeGzipSqlStatements(temporaryTopiaApplicationContext, "ImportData_" + importDataSourceConfiguration.getLabel(), dataDump); + + if (!referantialImported) { + + if (log.isInfoEnabled()) { + log.info("Import referential."); + } + + // Réplication du référentiel + replicateReferential(temporaryTopiaApplicationContext, topiaApplicationContext); + + } if (log.isInfoEnabled()) { log.info("Import data."); } - DataSourceDumpProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, DataSourceDumpProducerService.class); - - byte[] dataDump = dumpProducerService.getDataDump(importDataIds); - - executeGzipSqlStatements(topiaApplicationContext, "ImportData_" + importDataSourceConfiguration.getLabel(), dataDump); + // Réplication des données + replicateData(temporaryTopiaApplicationContext, topiaApplicationContext, importDataIds); } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java b/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java index 4b13c45..2dc55a7 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java @@ -1,14 +1,19 @@ package fr.ird.observe.services; +import fr.ird.observe.ObserveTopiaApplicationContext; +import fr.ird.observe.ObserveTopiaPersistenceContext; import fr.ird.observe.services.configuration.ObserveDataSourceConfiguration; +import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaSupport; import fr.ird.observe.services.dto.constants.ReferentialLocale; import fr.ird.observe.test.TestHelper; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.junit.Assert; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; +import org.nuiton.topia.persistence.TopiaException; import org.nuiton.util.version.Version; import java.io.File; @@ -125,4 +130,49 @@ public class ApplicationContextResource implements TestRule { public void setTestDirectory(File testDirectory) { this.testDirectory = testDirectory; } + + public ObserveDataSourceConfigurationTopiaH2 createDataSourceConfigurationH2(Class<?> testClass, String methodName) { + + ObserveDataSourceConfigurationTopiaH2 configurationTopiaH2 = new ObserveDataSourceConfigurationTopiaH2(); + + File localDbFile = new File(getTestDirectory(), methodName); + + if (log.isInfoEnabled()) { + log.info("db directory: " + localDbFile); + } + configurationTopiaH2.setLabel(testClass.getSimpleName() + "#" + methodName); + configurationTopiaH2.setUsername(TestHelper.TEST_H2_LOGIN); + configurationTopiaH2.setPassword(TestHelper.TEST_H2_PASSWORD); + configurationTopiaH2.setDirectory(localDbFile); + configurationTopiaH2.setDbName("obstuna"); + configurationTopiaH2.setAutoMigrate(true); + return configurationTopiaH2; + + } + + public void assertSchemaCreated(ObserveTopiaApplicationContext topiaApplicationContext) { + + try (ObserveTopiaPersistenceContext topiaPersistenceContext = topiaApplicationContext.newPersistenceContext()) { + TestHelper.assertSchemaCreated(topiaApplicationContext, topiaPersistenceContext, false); + } + + } + + public void assertEntitiesExist(ObserveTopiaApplicationContext topiaApplicationContext, String... ids) { + + try (ObserveTopiaPersistenceContext persistenceContext = topiaApplicationContext.newPersistenceContext()) { + + for (String id : ids) { + try { + persistenceContext.findByTopiaId(id); + if (log.isInfoEnabled()) { + log.info("Entity " + id + " found."); + } + } catch (TopiaException e) { + Assert.fail("Could not find entity with id: " + id); + } + } + } + } + } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java b/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java index 95511b0..f3ec450 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java @@ -96,13 +96,13 @@ public class DataSourceResource implements TestRule { } - public <E extends TopiaEntity> boolean exists(Class<E> entityType, String id) { + public <E extends TopiaEntity> boolean exists(Class<E> entityType, String id) { TopiaPersistenceContext persistenceContext = newPersistenceContext(); TopiaDao<E> dao = persistenceContext.getDao(entityType); - boolean exist = dao.forTopiaIdEquals(id).exists(); + boolean exist = dao.forTopiaIdEquals(id).exists(); return exist; @@ -145,28 +145,10 @@ public class DataSourceResource implements TestRule { } - dataSourceConfiguration = createDataSourceConfigurationH2(testClass, methodName); + dataSourceConfiguration = applicationContextResource.createDataSourceConfigurationH2(testClass, methodName); } - protected ObserveDataSourceConfigurationTopiaH2 createDataSourceConfigurationH2(Class<?> testClass, String methodName) { - - ObserveDataSourceConfigurationTopiaH2 configurationTopiaH2 = new ObserveDataSourceConfigurationTopiaH2(); - - File localDbFile = new File(applicationContextResource.getTestDirectory(), "localdb"); - - if (log.isInfoEnabled()) { - log.info("db directory: " + localDbFile); - } - configurationTopiaH2.setLabel(testClass.getSimpleName() + "#" + methodName); - configurationTopiaH2.setUsername(TestHelper.TEST_H2_LOGIN); - configurationTopiaH2.setPassword(TestHelper.TEST_H2_PASSWORD); - configurationTopiaH2.setDirectory(localDbFile); - configurationTopiaH2.setDbName("obstuna"); - configurationTopiaH2.setAutoMigrate(true); - return configurationTopiaH2; - - } protected void after(Description description) { diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java index 4eb5c36..9de9d68 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java @@ -41,11 +41,20 @@ public class DataSourceDumpProducerServiceTopiaTest extends AbstractServiceTopia } + @Test public void testGetDataDump() throws Exception { - byte[] referentialDump = service.getDataDump(); - Assert.assertNotNull(referentialDump); + byte[] dataDump = service.getDataDump(ImmutableSet.of(TRIP_SEINE_ID_1)); + Assert.assertNotNull(dataDump); + + } + + @Test + public void testGetAllDataDump() throws Exception { + + byte[] allDataDump = service.getAllDataDump(); + Assert.assertNotNull(allDataDump); } } \ No newline at end of file diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java index 4441b80..a5cb968 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java @@ -2,18 +2,16 @@ package fr.ird.observe.services.service; import com.google.common.collect.ImmutableSet; import fr.ird.observe.ObserveTopiaApplicationContext; -import fr.ird.observe.ObserveTopiaPersistenceContext; import fr.ird.observe.entities.migration.H2DataSourceMigration; import fr.ird.observe.services.ApplicationContextResource; import fr.ird.observe.services.DataSourceResource; +import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; -import fr.ird.observe.test.TestHelper; import org.junit.Before; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -64,7 +62,7 @@ public class DataSourceServiceTopiaTest { service.create(dataSourceCreateConfiguration); service.open(); - assertSchemaCreated(); + applicationContextResource.assertSchemaCreated(dataSourceResource.getTopiaApplicationContext()); } @@ -75,7 +73,7 @@ public class DataSourceServiceTopiaTest { dataSourceCreateConfiguration.setCanCreateEmptyDatabase(true); service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + applicationContextResource.assertSchemaCreated(dataSourceResource.getTopiaApplicationContext()); } @@ -90,50 +88,56 @@ public class DataSourceServiceTopiaTest { dataSourceCreateConfiguration.setImportDatabase(dumpContent); service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + applicationContextResource.assertSchemaCreated(dataSourceResource.getTopiaApplicationContext()); } - //TODO - @Ignore @Test public void testCreateDataSourceFromImportReferential() throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException { - String scriptPath = applicationContextResource.getScriptPath("referentiel", H2DataSourceMigration.V_4_0); - byte[] dumpContent = dataSourceResource.getDatabaseToImportContent(scriptPath); + // On crée un base qui fait office de source de référentiel + String scriptPath = applicationContextResource.getScriptPath("referentiel", H2DataSourceMigration.V_4_0_1); + dataSourceResource.createDataSourceFromScript(scriptPath); + + ObserveDataSourceConfigurationTopiaH2 dataSourceConfiguration = dataSourceResource.getDataSourceConfiguration(); DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); - dataSourceCreateConfiguration.setImportDatabase(dumpContent); + dataSourceCreateConfiguration.setImportReferentialDataSourceConfiguration(dataSourceConfiguration); - service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + ObserveDataSourceConfigurationTopiaH2 createdDatabaseDataSourceConfiguration = applicationContextResource.createDataSourceConfigurationH2(getClass(), "resultDatabase"); + + DataSourceService dataSourceService = applicationContextResource.newService(createdDatabaseDataSourceConfiguration, DataSourceService.class); + + dataSourceService.create(dataSourceCreateConfiguration); + + ObserveTopiaApplicationContext createdDatabaseTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(createdDatabaseDataSourceConfiguration); + applicationContextResource.assertSchemaCreated(createdDatabaseTopiaApplicationContext); } - //TODO - @Ignore @Test public void testCreateDataSourceFromImportReferentialAndData() throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException { - String scriptPath = applicationContextResource.getScriptPath("referentiel", H2DataSourceMigration.V_4_0); - byte[] dumpContent = dataSourceResource.getDatabaseToImportContent(scriptPath); + // On crée un base qui fait office de source de référentiel et de source de données (uniquement des données seine) + String scriptPath = applicationContextResource.getScriptPath("dataForTestSeine", H2DataSourceMigration.V_4_0_1); + dataSourceResource.createDataSourceFromScript(scriptPath); + + ObserveDataSourceConfigurationTopiaH2 dataSourceConfiguration = dataSourceResource.getDataSourceConfiguration(); DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); - dataSourceCreateConfiguration.setImportDatabase(dumpContent); + dataSourceCreateConfiguration.setImportReferentialDataSourceConfiguration(dataSourceConfiguration); + dataSourceCreateConfiguration.setImportDataConfiguration(dataSourceConfiguration, ImmutableSet.of(AbstractServiceTopiaTest.TRIP_SEINE_ID_1)); - service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + ObserveDataSourceConfigurationTopiaH2 createdDatabaseDataSourceConfiguration = applicationContextResource.createDataSourceConfigurationH2(getClass(), "resultDatabase"); - } + DataSourceService dataSourceService = applicationContextResource.newService(createdDatabaseDataSourceConfiguration, DataSourceService.class); - protected void assertSchemaCreated() { + dataSourceService.create(dataSourceCreateConfiguration); - ObserveTopiaApplicationContext topiaApplicationContext = dataSourceResource.getTopiaApplicationContext(); - try (ObserveTopiaPersistenceContext topiaPersistenceContext = topiaApplicationContext.newPersistenceContext()) { - TestHelper.assertSchemaCreated(topiaApplicationContext, topiaPersistenceContext, false); - } + ObserveTopiaApplicationContext createdDatabaseTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(createdDatabaseDataSourceConfiguration); + applicationContextResource.assertSchemaCreated(createdDatabaseTopiaApplicationContext); + applicationContextResource.assertEntitiesExist(createdDatabaseTopiaApplicationContext, AbstractServiceTopiaTest.TRIP_SEINE_ID_1); } - } \ No newline at end of file -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@list.forge.codelutin.com>.