This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository tutti. See http://git.codelutin.com/tutti.git commit 46019f827f4a20538ba65b19da8627089a79f1ae Author: Tony CHEMIT <chemit@codelutin.com> Date: Sat Jan 9 13:12:08 2016 +0100 Amélioration d'exécution d'un script sql --- .../observe/ObserveTopiaApplicationContext.java | 49 +++------- .../observe/ObserveTopiaPersistenceContext.java | 10 +- .../java/fr/ird/observe/RunScriptTopiaSqlWork.java | 103 +++++++++++++++++++++ .../services/service/DataSourceServiceTopia.java | 31 +------ 4 files changed, 130 insertions(+), 63 deletions(-) diff --git a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java index 2a9ee57..9735743 100644 --- a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java +++ b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaApplicationContext.java @@ -25,7 +25,6 @@ package fr.ird.observe; import com.google.common.collect.ImmutableSet; import fr.ird.observe.entities.Entities; import fr.ird.observe.entities.migration.ObserveMigrationEngine; -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hibernate.HibernateException; @@ -38,21 +37,15 @@ import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity; import org.nuiton.topia.service.sql.batch.TopiaSqlBatchService; import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTablesFactory; -import org.nuiton.util.GZUtil; import org.nuiton.util.StringUtil; -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.StringWriter; import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; import java.util.UUID; -import java.util.zip.GZIPInputStream; public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicationContext { @@ -135,10 +128,19 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat } } - public void executeSqlStatements(byte... content) throws IOException { + public void executeSqlStatements(byte... content) { - if (getConfiguration().isH2Configuration()) { + try { + executeSqlStatements0(content); + } catch (IOException e) { + throw new TopiaException(e); + } + + } + + protected void executeSqlStatements0(byte... content) throws IOException { + if (getConfiguration().isH2Configuration()) { Path tempFile = Files.createTempFile("observeRestoreH2", ".sql"); log.info(String.format("Restore script size: %s to h2 from file : %s", StringUtil.convertMemory(content.length), tempFile.toString())); @@ -152,36 +154,11 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat } else { - boolean gzip; - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(content)) { - gzip = GZUtil.isGzipStream(byteArrayInputStream); - } - - String sqlScript; - - try (StringWriter writer = new StringWriter()) { - if (gzip) { - - try (InputStreamReader inputSream = new InputStreamReader(new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(content))))) { - IOUtils.copy(inputSream, writer); - } - sqlScript = writer.toString(); - log.info(String.format("Gunzip script %s to %s", StringUtil.convertMemory(content.length), StringUtil.convertMemory(sqlScript.length()))); - - } else { - - try (InputStreamReader inputSream = new InputStreamReader(new BufferedInputStream(new ByteArrayInputStream(content)))) { - IOUtils.copy(inputSream, writer); - } - sqlScript = writer.toString(); - log.info(String.format("Load script (size: %s)", StringUtil.convertMemory(sqlScript.length()))); - } - - } + log.info(String.format("Load script (size: %s)", StringUtil.convertMemory(content.length))); try (ObserveTopiaPersistenceContext persistenceContext = newPersistenceContext()) { - persistenceContext.getSqlSupport().executeSql(sqlScript); + persistenceContext.executeSqlScript(content); persistenceContext.commit(); } } diff --git a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java index 7e2cf3c..32e3f6e 100644 --- a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java +++ b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java @@ -32,6 +32,7 @@ import org.nuiton.topia.persistence.TopiaDao; import org.nuiton.topia.persistence.internal.AbstractTopiaPersistenceContextConstructorParameter; import org.nuiton.topia.persistence.support.TopiaSqlQuery; +import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -40,7 +41,9 @@ import java.util.Date; public class ObserveTopiaPersistenceContext extends AbstractObserveTopiaPersistenceContext { - /** Logger. */ + /** + * Logger. + */ private static final Log log = LogFactory.getLog(ObserveTopiaPersistenceContext.class); public ObserveTopiaPersistenceContext(AbstractTopiaPersistenceContextConstructorParameter parameter) { @@ -126,4 +129,9 @@ public class ObserveTopiaPersistenceContext extends AbstractObserveTopiaPersiste return set.getLong(1); } } + + public void executeSqlScript(byte... content) throws IOException { + + getSqlSupport().doSqlWork(new RunScriptTopiaSqlWork(1000, content)); + } } diff --git a/observe-entities/src/main/java/fr/ird/observe/RunScriptTopiaSqlWork.java b/observe-entities/src/main/java/fr/ird/observe/RunScriptTopiaSqlWork.java new file mode 100644 index 0000000..fe74133 --- /dev/null +++ b/observe-entities/src/main/java/fr/ird/observe/RunScriptTopiaSqlWork.java @@ -0,0 +1,103 @@ +package fr.ird.observe; + +import org.nuiton.topia.persistence.TopiaException; +import org.nuiton.topia.persistence.support.TopiaSqlWork; +import org.nuiton.util.GZUtil; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.zip.GZIPInputStream; + +/** + * TODO Move this in ToPIA. + * + * Created on 09/01/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class RunScriptTopiaSqlWork implements TopiaSqlWork { + + protected final byte[] content; + protected final boolean gzip; + protected final int batchSize; + + public RunScriptTopiaSqlWork(int batchSize, byte[] content) throws IOException { + this.batchSize = batchSize; + this.content = content; + + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(content)) { + gzip = GZUtil.isGzipStream(byteArrayInputStream); + } + } + + @Override + public void execute(Connection connection) throws SQLException { + + boolean autoCommit = connection.getAutoCommit(); + + try { + connection.setAutoCommit(false); + try (BufferedReader reader = createReader()) { + + try (Statement statement = connection.createStatement()) { + + int batchSize = 0; + String command = null; + String next; + while ((next = reader.readLine()) != null) { + + String trimLine = next.trim(); + + if (trimLine.startsWith("--")) { + continue; + } + + if (command == null) { + command = next; + } else { + command += ' ' + next; + } + if (trimLine.endsWith(";")) { + statement.addBatch(command); + batchSize++; + command = null; + + if (batchSize % this.batchSize == 0) { + flushStatement(statement); + } + } + + } + + flushStatement(statement); + } + + } catch (IOException e) { + throw new TopiaException(e); + } + } finally { + connection.setAutoCommit(autoCommit); + } + } + + protected void flushStatement(Statement statement) throws SQLException { + + statement.executeBatch(); + statement.clearBatch(); + + } + + protected BufferedReader createReader() { + try { + return new BufferedReader(new InputStreamReader(new BufferedInputStream(gzip ? new GZIPInputStream(new ByteArrayInputStream(content)) : new ByteArrayInputStream(content)))); + } catch (IOException e) { + throw new TopiaException(e); + } + } +} diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java index f679560..0e8d482 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java @@ -51,7 +51,6 @@ import org.nuiton.topia.persistence.jdbc.JdbcHelper; import org.nuiton.util.version.Version; import java.io.File; -import java.io.IOException; import java.util.Set; import static org.nuiton.i18n.I18n.l; @@ -168,8 +167,7 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS } byte[] importDatabase = dataSourceCreateConfiguration.getImportDatabase(); - executeGzipSqlStatements(topiaApplicationContext, importDatabase); - + topiaApplicationContext.executeSqlStatements(importDatabase); topiaApplicationContext.getMigrationService().runSchemaMigration(); } else { @@ -198,11 +196,9 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS } ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); - - // Récupération du dump qui contient le référentiel SqlScriptProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, SqlScriptProducerService.class); byte[] referentialDump = dumpProducerService.getReferentialDump(); - executeGzipSqlStatements(topiaApplicationContext, referentialDump); + topiaApplicationContext.executeSqlStatements(referentialDump); referantialImported = true; @@ -226,7 +222,7 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS if (log.isInfoEnabled()) { log.info("Import referential."); } - executeGzipSqlStatements(topiaApplicationContext, referentialDump); + topiaApplicationContext.executeSqlStatements(referentialDump); } @@ -242,7 +238,7 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS if (log.isInfoEnabled()) { log.info("Import data."); } - executeGzipSqlStatements(topiaApplicationContext, dataDump); + topiaApplicationContext.executeSqlStatements(dataDump); } @@ -355,29 +351,12 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS @Override public void importData(ImportDataDto importData) { - // création d'une base temporaire pour y importer le référentiel et les données -// ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = createTemporaryDatabaseFromDump("ImportData", importData.getDumpData()); -// ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); - if (log.isInfoEnabled()) { log.info("Import data."); } - // Réplication des données ObserveTopiaApplicationContext topiaApplicationContext = serviceContext.getTopiaApplicationContext(); - executeGzipSqlStatements(topiaApplicationContext, importData.getDumpData()); -// replicateData(temporaryTopiaApplicationContext, topiaApplicationContext, importData.getImportDataIds()); - - } - - protected void executeGzipSqlStatements(ObserveTopiaApplicationContext topiaApplicationContext, byte... content) { - - try { - topiaApplicationContext.executeSqlStatements(content); - } catch (IOException e) { - //TODO Avoir une exception concrete - throw new RuntimeException("Could not execute GZipSqlStatements", e); - } + topiaApplicationContext.executeSqlStatements(importData.getDumpData()); } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.