Tony CHEMIT pushed to branch develop-7.x at ultreiaio / ird-observe

Commits:

5 changed files:

Changes:

  • common-persistence/pom.xml
    ... ... @@ -47,6 +47,10 @@
    47 47 
           <groupId>org.nuiton.topia</groupId>
    48 48 
           <artifactId>topia-persistence</artifactId>
    49 49 
         </dependency>
    50 
    +    <dependency>
    51 
    +      <groupId>io.ultreia.java4all.topia</groupId>
    52 
    +      <artifactId>persistence</artifactId>
    53 
    +    </dependency>
    50 54 
         <dependency>
    51 55 
           <groupId>io.ultreia.java4all.topia</groupId>
    52 56 
           <artifactId>service-migration</artifactId>

  • common-persistence/src/main/java/fr/ird/observe/entities/BlobIdsIterator.java
    1 
    +package fr.ird.observe.entities;
    2 
    +
    3 
    +/*-
    4 
    + * #%L
    5 
    + * ObServe Toolkit :: Common Persistence
    6 
    + * %%
    7 
    + * Copyright (C) 2008 - 2020 IRD, Code Lutin, Ultreia.io
    8 
    + * %%
    9 
    + * This program is free software: you can redistribute it and/or modify
    10 
    + * it under the terms of the GNU General Public License as
    11 
    + * published by the Free Software Foundation, either version 3 of the
    12 
    + * License, or (at your option) any later version.
    13 
    + *
    14 
    + * This program is distributed in the hope that it will be useful,
    15 
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    16 
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    17 
    + * GNU General Public License for more details.
    18 
    + *
    19 
    + * You should have received a copy of the GNU General Public
    20 
    + * License along with this program.  If not, see
    21 
    + * <http://www.gnu.org/licenses/gpl-3.0.html>.
    22 
    + * #L%
    23 
    + */
    24 
    +
    25 
    +import org.nuiton.topia.persistence.TopiaApplicationContext;
    26 
    +import org.nuiton.topia.persistence.jdbc.JdbcPostgresHelper;
    27 
    +import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity;
    28 
    +import org.nuiton.topia.persistence.metadata.TopiaMetadataModel;
    29 
    +
    30 
    +import java.io.BufferedReader;
    31 
    +import java.io.BufferedWriter;
    32 
    +import java.io.Closeable;
    33 
    +import java.io.IOException;
    34 
    +import java.nio.file.Files;
    35 
    +import java.nio.file.Path;
    36 
    +import java.sql.PreparedStatement;
    37 
    +import java.sql.ResultSet;
    38 
    +import java.sql.SQLException;
    39 
    +import java.util.Iterator;
    40 
    +import java.util.Objects;
    41 
    +
    42 
    +public class BlobIdsIterator implements Iterator<String>, Closeable {
    43 
    +
    44 
    +    private final TopiaMetadataModel model;
    45 
    +
    46 
    +    private final Path cachePath;
    47 
    +    private final JdbcPostgresHelper jdbcHelper;
    48 
    +    private FileIterator iterator;
    49 
    +
    50 
    +    public BlobIdsIterator(TopiaMetadataModel model, Path cachePath, TopiaApplicationContext<?> applicationContext) {
    51 
    +        this.model = Objects.requireNonNull(model);
    52 
    +        this.cachePath = Objects.requireNonNull(cachePath);
    53 
    +        this.jdbcHelper = new JdbcPostgresHelper(Objects.requireNonNull(applicationContext).getConfiguration());
    54 
    +    }
    55 
    +
    56 
    +    @Override
    57 
    +    public boolean hasNext() {
    58 
    +        return iterator().hasNext();
    59 
    +    }
    60 
    +
    61 
    +    @Override
    62 
    +    public String next() {
    63 
    +        return iterator().next();
    64 
    +    }
    65 
    +
    66 
    +    public FileIterator iterator() {
    67 
    +        if (iterator == null) {
    68 
    +            iterator = new FileIterator(cachePath, model, jdbcHelper);
    69 
    +        }
    70 
    +        return iterator;
    71 
    +    }
    72 
    +
    73 
    +    @Override
    74 
    +    public void close() throws IOException {
    75 
    +        if (iterator != null) {
    76 
    +            iterator.close();
    77 
    +            iterator = null;
    78 
    +        }
    79 
    +    }
    80 
    +
    81 
    +    private static class FileIterator implements Iterator<String>, Closeable {
    82 
    +        private final Iterator<String> iterator;
    83 
    +        private final BufferedReader bufferedReader;
    84 
    +
    85 
    +        public FileIterator(Path cachePath, TopiaMetadataModel model, JdbcPostgresHelper jdbcHelper) {
    86 
    +            if (Files.notExists(cachePath)) {
    87 
    +                try {
    88 
    +                    createCache(jdbcHelper, model, cachePath);
    89 
    +                } catch (IOException e) {
    90 
    +                    throw new IllegalStateException("can't create cache at " + cachePath, e);
    91 
    +                }
    92 
    +            }
    93 
    +            try {
    94 
    +                bufferedReader = Files.newBufferedReader(Objects.requireNonNull(cachePath));
    95 
    +            } catch (IOException e) {
    96 
    +                throw new IllegalStateException("can't create reader on " + cachePath, e);
    97 
    +            }
    98 
    +            iterator = bufferedReader.lines().iterator();
    99 
    +        }
    100 
    +
    101 
    +        @Override
    102 
    +        public boolean hasNext() {
    103 
    +            return iterator.hasNext();
    104 
    +        }
    105 
    +
    106 
    +        @Override
    107 
    +        public String next() {
    108 
    +            return iterator.next();
    109 
    +        }
    110 
    +
    111 
    +        @Override
    112 
    +        public void close() throws IOException {
    113 
    +            bufferedReader.close();
    114 
    +        }
    115 
    +
    116 
    +        private void createCache(JdbcPostgresHelper jdbcHelper, TopiaMetadataModel model, Path cachePath) throws IOException {
    117 
    +            if (Files.notExists(cachePath.getParent())) {
    118 
    +                Files.createDirectory(cachePath.getParent());
    119 
    +            }
    120 
    +            try (BufferedWriter writer = Files.newBufferedWriter(cachePath)) {
    121 
    +                for (TopiaMetadataEntity metadataEntity : model) {
    122 
    +                    if (metadataEntity.withBlob()) {
    123 
    +                        for (String blobProperty : metadataEntity.getBlobProperties()) {
    124 
    +                            fillCache(jdbcHelper, metadataEntity.getDbSchemaName(), metadataEntity.getDbTableName(), metadataEntity.getDbColumnName(blobProperty), writer);
    125 
    +                        }
    126 
    +                    }
    127 
    +                }
    128 
    +                writer.flush();
    129 
    +            }
    130 
    +        }
    131 
    +
    132 
    +        private void fillCache(JdbcPostgresHelper jdbcHelper, String dbSchemaName, String dbTableName, String dbColumnName, BufferedWriter writer) {
    133 
    +            String sql = String.format("SELECT %1$s FROM %2$s.%3$s WHERE %1$s IS NOT NULL ORDER BY topiaId ASC", dbColumnName, dbSchemaName, dbTableName);
    134 
    +            jdbcHelper.consume(c -> {
    135 
    +                try (PreparedStatement preparedStatement = c.prepareStatement(sql)) {
    136 
    +                    try (ResultSet resultSet = preparedStatement.executeQuery()) {
    137 
    +                        while (resultSet.next()) {
    138 
    +                            writer.write(resultSet.getString(1));
    139 
    +                            writer.newLine();
    140 
    +                        }
    141 
    +                    }
    142 
    +                } catch (SQLException | IOException e) {
    143 
    +                    throw new IllegalStateException(e);
    144 
    +                }
    145 
    +            });
    146 
    +        }
    147 
    +    }
    148 
    +}

  • persistence/src/main/java/fr/ird/observe/persistence/ObserveTopiaApplicationContext.java
    ... ... @@ -23,6 +23,7 @@ package fr.ird.observe.persistence;
    23 23 
      */
    24 24
    25 25 
     import com.google.common.collect.ImmutableSet;
    26 
    +import fr.ird.observe.entities.BlobIdsIterator;
    26 27 
     import fr.ird.observe.entities.referentiel.ObserveReferentialEntity;
    27 28 
     import org.apache.logging.log4j.LogManager;
    28 29 
     import org.apache.logging.log4j.Logger;
    ... ... @@ -46,6 +47,7 @@ import org.nuiton.topia.service.script.table.TopiaSqlTable;
    46 47 
     import org.nuiton.topia.service.script.table.TopiaSqlTables;
    47 48 
     import org.nuiton.topia.service.script.table.TopiaSqlTablesFactory;
    48 49
    50 
    +import java.nio.file.Path;
    49 51 
     import java.util.EnumSet;
    50 52 
     import java.util.LinkedHashSet;
    51 53 
     import java.util.List;
    ... ... @@ -335,6 +337,9 @@ public class ObserveTopiaApplicationContext extends AbstractObserveTopiaApplicat
    335 337 
             return referentialTables = topiaSqlTablesFactory.newReplicateEntityTables(new TripReplicateTablesPredicate(), entityEnum);
    336 338 
         }
    337 339
    340 
    +    public BlobIdsIterator newBlobIdsIterator(Path blobIdsPath) {
    341 
    +        return new BlobIdsIterator(getMetadataModel(), blobIdsPath, this);
    342 
    +    }
    338 343
    339 344 
         private static class TripReplicateTablesPredicate implements TopiaSqlTablesFactory.TopiaSqlTablesPredicate {
    340 345

  • services-local/src/main/java/fr/ird/observe/services/local/ObserveSecurityHelper.java
    ... ... @@ -26,8 +26,9 @@ import com.google.common.collect.ImmutableSet;
    26 26 
     import com.google.common.collect.Iterables;
    27 27 
     import fr.ird.observe.dto.ObserveDbRole;
    28 28 
     import fr.ird.observe.dto.db.ObserveDbUserDto;
    29 
    +import fr.ird.observe.entities.BlobIdsIterator;
    29 30 
     import fr.ird.observe.persistence.Entities;
    30 
    -import fr.ird.observe.persistence.ObserveTopiaConfiguration;
    31 
    +import fr.ird.observe.persistence.ObserveTopiaApplicationContext;
    31 32 
     import org.apache.commons.lang3.tuple.Pair;
    32 33 
     import org.apache.logging.log4j.LogManager;
    33 34 
     import org.apache.logging.log4j.Logger;
    ... ... @@ -49,6 +50,7 @@ import java.util.Collections;
    49 50 
     import java.util.HashSet;
    50 51 
     import java.util.LinkedHashSet;
    51 52 
     import java.util.List;
    53 
    +import java.util.Objects;
    52 54 
     import java.util.Set;
    53 55 
     import java.util.function.Function;
    54 56 
     import java.util.stream.Collectors;
    ... ... @@ -63,7 +65,11 @@ public class ObserveSecurityHelper {
    63 65 
         private static final String OBSERVE_LONGLINE_SCHEMA_NAME = "observe_longline";
    64 66 
         private static final Function<String, String> ESCAPE_STRING = input -> "\"" + input + "\"";
    65 67 
         private static final String REVOKE_ON_TABLE_ALL_PATTERN = "REVOKE ALL ON %s.%s FROM %s CASCADE;";
    68 
    +    private static final String REVOKE_ON_LARGE_OBJECT_ALL_PATTERN = "REVOKE ALL ON LARGE OBJECT %s FROM %s CASCADE;";
    66 69 
         private static final String SET_ON_TABLE_OWNER_PATTERN = "ALTER TABLE %s.%s OWNER TO %s;";
    70 
    +    private static final String SET_ON_LARGE_OBJECT_OWNER_PATTERN = "ALTER LARGE OBJECT %s OWNER TO %s;";
    71 
    +    private static final String GRANT_ON_LARGE_OBJECT_READ_PATTERN = "GRANT SELECT ON LARGE OBJECT %s TO %s;";
    72 
    +    private static final String GRANT_ON_LARGE_OBJECT_ALL_PATTERN = "GRANT ALL ON LARGE OBJECT %s TO %s;";
    67 73 
         private static final String GRANT_ON_TABLE_READ_PATTERN = "GRANT SELECT ON %s.%s TO %s;";
    68 74 
         private static final String GRANT_ON_TABLE_ALL_PATTERN = "GRANT ALL ON %s.%s TO %s;";
    69 75 
         private static final String GRANT_ON_FUNCTION_PATTERN = "GRANT EXECUTE ON FUNCTION %s TO %s;";
    ... ... @@ -74,24 +80,26 @@ public class ObserveSecurityHelper {
    74 80 
                 TMSVersionHibernateDao.TABLE_NAME,
    75 81 
                 TMSVersionHibernateDao.LEGACY_TABLE_NAME);
    76 82 
         private static final Set<String> FUNCTION_NAMES_PREFIXS = ImmutableSet.of("ST_MakePoint",
    77 
    -            "ST_SetSRID",
    78 
    -            "sync_",
    79 
    -            "tr_sync",
    80 
    -            "ot_enhanced_school_type",
    81 
    -            "observe_");
    83 
    +                                                                              "ST_SetSRID",
    84 
    +                                                                              "sync_",
    85 
    +                                                                              "tr_sync",
    86 
    +                                                                              "ot_enhanced_school_type",
    87 
    +                                                                              "observe_");
    82 88 
         private static final String SCHEMA_PUBLIC = "public";
    83 89 
         private static final Set<String> SCHEMAS = ImmutableSet.of(SCHEMA_PUBLIC,
    84 
    -            OBSERVE_COMMON_SCHEMA_NAME,
    85 
    -            OBSERVE_SEINE_SCHEMA_NAME,
    86 
    -            OBSERVE_LONGLINE_SCHEMA_NAME);
    90 
    +                                                               OBSERVE_COMMON_SCHEMA_NAME,
    91 
    +                                                               OBSERVE_SEINE_SCHEMA_NAME,
    92 
    +                                                               OBSERVE_LONGLINE_SCHEMA_NAME);
    87 93 
         private static final Logger log = LogManager.getLogger(ObserveSecurityHelper.class);
    88 94 
         private final JdbcPostgresHelper jdbcHelper;
    89 95 
         private final Path temporaryDirectory;
    96 
    +    private final ObserveTopiaApplicationContext applicationContext;
    90 97
    91 98
    92 
    -    public ObserveSecurityHelper(ObserveTopiaConfiguration jdbcConfiguration) {
    93 
    -        this.jdbcHelper = new JdbcPostgresHelper(jdbcConfiguration);
    94 
    -        this.temporaryDirectory = jdbcConfiguration.getTemporaryDirectory();
    99 
    +    public ObserveSecurityHelper(ObserveTopiaApplicationContext applicationContext) {
    100 
    +        this.applicationContext = Objects.requireNonNull(applicationContext);
    101 
    +        this.jdbcHelper = new JdbcPostgresHelper(applicationContext.getConfiguration());
    102 
    +        this.temporaryDirectory = applicationContext.getConfiguration().getTemporaryDirectory();
    95 103 
         }
    96 104
    97 105 
         public void applySecurity(Set<ObserveDbUserDto> users) {
    ... ... @@ -104,10 +112,11 @@ public class ObserveSecurityHelper {
    104 112 
             } catch (IOException e) {
    105 113 
                 throw new IllegalStateException("Can't create temporary path", e);
    106 114 
             }
    115 
    +        Path blobIdsPath = scriptPath.getParent().resolve(scriptPath.toFile().getName().replace(".sql",".blob-ids"));
    107 116
    108 117 
             try {
    109 118 
                 try (SqlScriptWriter sqlScriptWriter = SqlScriptWriter.of(scriptPath)) {
    110 
    -                createSecurityScript(users, sqlScriptWriter);
    119 
    +                createSecurityScript(users, sqlScriptWriter, blobIdsPath);
    111 120 
                     log.info(String.format("Generate security script %d statements(s) at %s", sqlScriptWriter.getStatementCount(), scriptPath));
    112 121 
                 }
    113 122 
                 jdbcHelper.consume(SqlScriptConsumer.of(scriptPath));
    ... ... @@ -116,7 +125,7 @@ public class ObserveSecurityHelper {
    116 125 
             }
    117 126 
         }
    118 127
    119 
    -    private void createSecurityScript(Set<ObserveDbUserDto> users, SqlScriptWriter sqlScriptWriter) {
    128 
    +    private void createSecurityScript(Set<ObserveDbUserDto> users, SqlScriptWriter sqlScriptWriter, Path blobIdsPath) throws IOException {
    120 129
    121 130 
             List<Pair<String, String>> tables = jdbcHelper.getTables(SCHEMAS, EXTRA_TABLES);
    122 131
    ... ... @@ -154,6 +163,7 @@ public class ObserveSecurityHelper {
    154 163 
             Set<String> referentialEscapedNames = escapedNames(referentialNames);
    155 164 
             Set<String> unusedEscapedNames = escapedNames(unusedNames);
    156 165
    166 
    +        BlobIdsIterator blobIdsIterator = applicationContext.newBlobIdsIterator(blobIdsPath);
    157 167
    158 168 
             // suppression de tous les droits
    159 169 
             {
    ... ... @@ -168,6 +178,7 @@ public class ObserveSecurityHelper {
    168 178 
                 addOnTablesForRole(REVOKE_ON_TABLE_ALL_PATTERN, sqlScriptWriter, tables, roles);
    169 179 
                 addOnSchemaForRole(REVOKE_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, SCHEMAS, roles);
    170 180 
                 addOnFunctionForRole(REVOKE_ON_FUNCTIONS_PATTERN, sqlScriptWriter, allPostgisFunctions, roles);
    181 
    +            addOnLargeObjectForRole(REVOKE_ON_LARGE_OBJECT_ALL_PATTERN, sqlScriptWriter, blobIdsIterator, roles);
    171 182
    172 183 
             }
    173 184
    ... ... @@ -175,6 +186,8 @@ public class ObserveSecurityHelper {
    175 186 
             addOnTablesForRole(SET_ON_TABLE_OWNER_PATTERN, sqlScriptWriter, tables, administratorEscapedName);
    176 187 
             addOnSchemaForRole(GRANT_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, SCHEMAS, administratorEscapedName);
    177 188 
             addOnSchemaForRole(GRANT_ON_FUNCTION_PATTERN, sqlScriptWriter, allPostgisFunctions, administratorEscapedName);
    189 
    +        addOnLargeObjectForRole(SET_ON_LARGE_OBJECT_OWNER_PATTERN, sqlScriptWriter, blobIdsIterator, administratorEscapedName);
    190 
    +        addOnLargeObjectForRole(GRANT_ON_LARGE_OBJECT_ALL_PATTERN, sqlScriptWriter, blobIdsIterator, administratorEscapedName);
    178 191
    179 192 
             // ajout administrateurs
    180 193 
             if (!technicalEscapedNames.isEmpty()) {
    ... ... @@ -182,6 +195,7 @@ public class ObserveSecurityHelper {
    182 195 
                 addOnTablesForRole(GRANT_ON_TABLE_ALL_PATTERN, sqlScriptWriter, tables, roles);
    183 196 
                 addOnSchemaForRole(GRANT_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, SCHEMAS, roles);
    184 197 
                 addOnSchemaForRole(GRANT_ON_FUNCTION_PATTERN, sqlScriptWriter, allPostgisFunctions, roles);
    198 
    +            addOnLargeObjectForRole(GRANT_ON_LARGE_OBJECT_ALL_PATTERN, sqlScriptWriter, blobIdsIterator, roles);
    185 199 
             }
    186 200
    187 201 
             // ajout utilisateur
    ... ... @@ -190,6 +204,7 @@ public class ObserveSecurityHelper {
    190 204 
                 addOnTablesForRole(GRANT_ON_TABLE_READ_PATTERN, sqlScriptWriter, tables, roles);
    191 205 
                 addOnSchemaForRole(GRANT_ON_SCHEMA_ALL_PATTERN, sqlScriptWriter, SCHEMAS, roles);
    192 206 
                 addOnSchemaForRole(GRANT_ON_FUNCTION_PATTERN, sqlScriptWriter, allPostgisFunctions, roles);
    207 
    +            addOnLargeObjectForRole(GRANT_ON_LARGE_OBJECT_READ_PATTERN, sqlScriptWriter, blobIdsIterator, roles);
    193 208 
             }
    194 209
    195 210 
             // ajout referentiel
    ... ... @@ -272,6 +287,18 @@ public class ObserveSecurityHelper {
    272 287 
             }
    273 288 
         }
    274 289
    290 
    +    private void addOnLargeObjectForRole(String pattern, SqlScriptWriter builder, BlobIdsIterator blobIdsIterator, String role) throws IOException {
    291 
    +        try {
    292 
    +            while (blobIdsIterator.hasNext()) {
    293 
    +                String blobId = blobIdsIterator.next();
    294 
    +                builder.writeSql(String.format(pattern, blobId, role));
    295 
    +            }
    296 
    +        } finally {
    297 
    +            blobIdsIterator.close();
    298 
    +        }
    299 
    +
    300 
    +    }
    301 
    +
    275 302 
         private void addOnFunctionForRole(String pattern, SqlScriptWriter builder, Set<String> functions, String role) {
    276 303 
             for (String t : functions) {
    277 304 
                 builder.writeSql(String.format(pattern, t, role));

  • services-local/src/main/java/fr/ird/observe/services/local/service/DataSourceServiceLocal.java
    ... ... @@ -582,11 +582,8 @@ public class DataSourceServiceLocal extends ObserveServiceLocal implements DataS
    582 582
    583 583 
                 ObserveDataSourceConfigurationTopiaPG sourceConfiguration = (ObserveDataSourceConfigurationTopiaPG) dataSourceConfiguration;
    584 584
    585 
    -            ObserveTopiaConfiguration topiaConfiguration;
    586 
    -            try (ObserveTopiaApplicationContext optionalTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(sourceConfiguration)) {
    587 
    -
    588 
    -                topiaConfiguration = optionalTopiaApplicationContext.getConfiguration();
    589 
    -                new ObserveSecurityHelper(topiaConfiguration).applySecurity(users);
    585 
    +            try (ObserveTopiaApplicationContext topiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(sourceConfiguration)) {
    586 
    +                new ObserveSecurityHelper(topiaApplicationContext).applySecurity(users);
    590 587 
                 }
    591 588
    592 589 
             }

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.