Author: tchemit Date: 2013-02-14 11:40:25 +0100 (Thu, 14 Feb 2013) New Revision: 412 Url: http://forge.codelutin.com/projects/tutti/repository/revisions/412 Log: improve synchro model Modified: trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeHelper.java trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeResult.java Modified: trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeHelper.java =================================================================== --- trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeHelper.java 2013-02-14 08:42:09 UTC (rev 411) +++ trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeHelper.java 2013-02-14 10:40:25 UTC (rev 412) @@ -28,7 +28,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import fr.ifremer.tutti.persistence.entities.TuttiEntities; -import org.apache.commons.lang.time.DateUtils; +import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hibernate.dialect.Dialect; @@ -121,6 +121,28 @@ if (result.isSuccess()) { + // prepare model (compute update date, count rows to update,...) + + for (TuttiTable tuttiTable : TuttiTable.values()) { + + String tableName = tuttiTable.name(); + + TuttiTableMetadata table = remoteMeta.getTable(tableName); + + if (log.isDebugEnabled()) { + log.debug("Prepare table: " + tableName); + } + prepareTable(table, + localConnection, + remoteConnection, + result); + } + + long totalRows = result.getTotalRows(); + if (log.isInfoEnabled()) { + log.info("Total rows to update: " + totalRows); + } + // prepare target (desactivate constraints) prepareSynch(localConnection); @@ -140,6 +162,14 @@ remoteConnection, result); } + if (log.isInfoEnabled()) { + long totalInserts = result.getTotalInserts(); + long totalUpdates = result.getTotalUpdates(); + log.info("Total rows to treat: " + totalRows); + log.info("Total rows inserted: " + totalInserts); + log.info("Total rows updated: " + totalUpdates); + log.info("Total rows treated: " + (totalInserts + totalUpdates)); + } } finally { releaseSynch(localConnection); } @@ -231,9 +261,9 @@ * if table does not use a updateDate columns or if there * is no data in table. */ - public Date getLastUpdateDate(Connection connection, - TuttiTableMetadata table) throws SQLException { - Date result = null; + public Timestamp getLastUpdateDate(Connection connection, + TuttiTableMetadata table) throws SQLException { + Timestamp result = null; if (table.isWithUpdateDateColumn()) { @@ -356,6 +386,7 @@ int columnCount = table.getColumnsCount(); String tableName = table.getName(); + String tablePrefix = getTablePrefix(table) + " - " + result.getNbRows(tableName); int[] pkIndex = table.getPkIndexs(); @@ -370,7 +401,6 @@ int countR = 0; int insertCount = 0; int updateCount = 0; - int nbRowsToTreat = result.getNbRows(tableName); while (incomingData.next()) { List<String> pk = getPk(incomingData, pkIndex); @@ -410,7 +440,7 @@ if (log.isDebugEnabled()) { String pkAsString = pk.toString(); - log.debug(String.format("[%s] Execute query %s (pk:%s)", tableName, sql, pkAsString)); + log.debug(String.format("%s Execute query %s (pk:%s)", tablePrefix, sql, pkAsString)); } statement.addBatch(); @@ -427,7 +457,7 @@ } if (countR % 10000 == 0) { if (log.isInfoEnabled()) { - log.info(String.format("[%s-%s] Rows Done: %s (inserts: %s, updates: %s)", tableName, nbRowsToTreat, countR, insertCount, updateCount)); + log.info(String.format("%s Done: %s (inserts: %s, updates: %s)", tablePrefix, countR, insertCount, updateCount)); } } } @@ -436,7 +466,7 @@ result.addInserts(tableName, insertCount); result.addUpdates(tableName, updateCount); if (log.isInfoEnabled()) { - log.info(String.format("[%s-%s] Rows Done: %s (inserts: %s, updates: %s)", tableName, nbRowsToTreat, countR, insertCount, updateCount)); + log.info(String.format("%s done: %s (inserts: %s, updates: %s)", tablePrefix, countR, insertCount, updateCount)); } if (insertCount > 0 && insertCount % 1000 != 0) { @@ -447,35 +477,29 @@ } if (log.isDebugEnabled()) { - log.debug(String.format("[%s] INSERT count: %s", tableName, result.getNbInserts(tableName))); - log.debug(String.format("[%s] UPDATE count: %s", tableName, result.getNbUpdates(tableName))); + log.debug(String.format("%s INSERT count: %s", tablePrefix, result.getNbInserts(tableName))); + log.debug(String.format("%s UPDATE count: %s", tablePrefix, result.getNbUpdates(tableName))); } } + protected String getTablePrefix(TuttiTableMetadata table) { + return "[" + table.getName() + "(" + ReferentialSynchronizeHelper.IS_ASSOCIATION_TABLE.apply(table) + ")]"; + + } + public void synchronizeTable(TuttiTableMetadata table, Connection localConnection, Connection remoteConnection, ReferentialSynchronizeResult result) throws SQLException { - // get last updateDate used by local db - Date updateDate = getLastUpdateDate(localConnection, table); - - if (updateDate != null) { - - // just inscrements of 1 milisecond to not having same - updateDate = new Timestamp(DateUtils.setMilliseconds(updateDate, 0).getTime()); - updateDate = new Timestamp(DateUtils.addSeconds(updateDate, 1).getTime()); - } String tableName = table.getName(); - long countToUpdate = - getCountDataToUpdate(remoteConnection, table, updateDate); + String tablePrefix = getTablePrefix(table); - if (log.isInfoEnabled()) { - log.info(String.format("[%s] count to update: %s", tableName, countToUpdate)); - } + // get last updateDate used by local db + Date updateDate = result.getUpdateDate(tableName); - result.addRows(tableName, (int) countToUpdate); + long countToUpdate = result.getNbRows(tableName); if (countToUpdate > 0) { @@ -490,7 +514,6 @@ // there is some data to update - TuttiTable tuttiTable = TuttiTable.valueOf(tableName); if (tuttiTable.isAssociation()) { PreparedStatement deleteStatement = @@ -498,7 +521,7 @@ "DELETE FROM " + tableName); deleteStatement.execute(); if (log.isInfoEnabled()) { - log.info("Delete from association: " + tableName); + log.info(tablePrefix + " Delete association table"); } } @@ -506,7 +529,7 @@ Set<List<String>> existingIds = getExistingPrimaryKeys(localConnection, table); if (log.isInfoEnabled()) { - log.info("[" + tableName + "] existingIds: " + existingIds.size()); + log.info(tablePrefix + " existingIds: " + existingIds.size()); } // update table content @@ -525,6 +548,36 @@ } + public void prepareTable(TuttiTableMetadata table, + Connection localConnection, + Connection remoteConnection, + ReferentialSynchronizeResult result) throws SQLException { + + String tablePrefix = getTablePrefix(table); + + String tableName = table.getName(); + + // get last updateDate used by local db + Timestamp updateDate = getLastUpdateDate(localConnection, table); + + if (updateDate != null) { + + // just inscrements of 1 milisecond to not having same + updateDate = new Timestamp(DateUtils.setMilliseconds(updateDate, 0).getTime()); + updateDate = new Timestamp(DateUtils.addSeconds(updateDate, 1).getTime()); + } + + long countToUpdate = + getCountDataToUpdate(remoteConnection, table, updateDate); + + if (log.isInfoEnabled()) { + log.info(String.format("%s nb rows to update: %s", tablePrefix, countToUpdate)); + } + + result.setUpdateDate(tableName, updateDate); + result.addRows(tableName, (int) countToUpdate); + } + public static void prepareSynch(Connection connection) throws SQLException { PreparedStatement statement = connection.prepareStatement("SET REFERENTIAL_INTEGRITY FALSE;"); statement.executeUpdate(); Modified: trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeResult.java =================================================================== --- trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeResult.java 2013-02-14 08:42:09 UTC (rev 411) +++ trunk/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/service/synchro/ReferentialSynchronizeResult.java 2013-02-14 10:40:25 UTC (rev 412) @@ -28,6 +28,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.sql.Timestamp; import java.util.Map; import java.util.Set; @@ -63,6 +64,13 @@ protected final Map<String, Integer> updateHits = Maps.newTreeMap(); /** + * timestamp of last update date (per table). + * + * @since 1.0 + */ + protected final Map<String, Timestamp> updateDateHits = Maps.newTreeMap(); + + /** * All table treated. * * @since 1.0 @@ -94,6 +102,30 @@ return ImmutableSet.copyOf(tableNames); } + public long getTotalRows() { + long result = 0; + for (Integer nb : rowHits.values()) { + result += nb; + } + return result; + } + + public long getTotalInserts() { + long result = 0; + for (Integer nb : insertHits.values()) { + result += nb; + } + return result; + } + + public long getTotalUpdates() { + long result = 0; + for (Integer nb : updateHits.values()) { + result += nb; + } + return result; + } + public int getNbRows(String tableName) { Integer result = rowHits.get(tableName); if (result == null) { @@ -136,6 +168,14 @@ } } + public Timestamp getUpdateDate(String tableName) { + return updateDateHits.get(tableName); + } + + public void setUpdateDate(String tableName, Timestamp t) { + updateDateHits.put(tableName, t); + } + public void addTableName(String tableName) { tableNames.add(tableName); }