branch feature/lucene created (now edbe89d)
This is an automated email from the git hooks/post-receive script. New change to branch feature/lucene in repository wao. See http://git.codelutin.com/wao.git at edbe89d Reprise des tests lucene de la banche lucene sur SVN This branch includes the following new commits: new bf80c88 fusion de deux migrations dans un même fichier suite à une fusion de branche de fonctionnalité new d2305ae Correction d'une clause catch pour un import CSV new 0e7cf12 Correction d'un WARN struts en ajoutant un champs manquant dans l'action new edbe89d Reprise des tests lucene de la banche lucene sur SVN The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit edbe89dd199254a0d764ebb170c69fe7a6e6aa5f Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:37:11 2014 +0100 Reprise des tests lucene de la banche lucene sur SVN commit 0e7cf1248e9b2d4c021834d779c93701c55c8292 Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:05:35 2014 +0100 Correction d'un WARN struts en ajoutant un champs manquant dans l'action commit d2305ae7858bd60907534017231e93f20eeffffd Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:05:13 2014 +0100 Correction d'une clause catch pour un import CSV commit bf80c884a6c39493a8058818cb93f8ae2fa8d242 Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 14:46:26 2014 +0100 fusion de deux migrations dans un même fichier suite à une fusion de branche de fonctionnalité -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/lucene in repository wao. See http://git.codelutin.com/wao.git commit bf80c884a6c39493a8058818cb93f8ae2fa8d242 Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 14:46:26 2014 +0100 fusion de deux migrations dans un même fichier suite à une fusion de branche de fonctionnalité --- ...pecies,_add_laboratories,_add_sclerochronology.sql} | 17 +++++++++++++++++ .../resources/db/migration/V5_0__sclerochronology.sql | 18 ------------------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/wao-persistence/src/main/resources/db/migration/V5_0__add_species,_add_laboratories.sql b/wao-persistence/src/main/resources/db/migration/V5_0__add_species,_add_laboratories,_add_sclerochronology.sql similarity index 73% rename from wao-persistence/src/main/resources/db/migration/V5_0__add_species,_add_laboratories.sql rename to wao-persistence/src/main/resources/db/migration/V5_0__add_species,_add_laboratories,_add_sclerochronology.sql index d53308d..96ab55c 100644 --- a/wao-persistence/src/main/resources/db/migration/V5_0__add_species,_add_laboratories.sql +++ b/wao-persistence/src/main/resources/db/migration/V5_0__add_species,_add_laboratories,_add_sclerochronology.sql @@ -57,3 +57,20 @@ alter table SampleRow rename column company to organisation; alter table WaoUser drop constraint fk42d03050f28824af; alter table WaoUser rename column company to organisation; + +-- Ouverture de l'accès au programme sclérochronologie du compte admin +insert into UserProfile values ( + 'fr.ifremer.wao.entities.UserProfile#129111142543000#933200987654321550', + 1, + now(), + (select topiaId from WaoUser where login = 'admin'), + 'SCLEROCHRONOLOGY', + 'ADMIN', + true, + uuid_in(md5(random()::text || now()::text)::cstring) +); + +alter table SampleRow add column species character varying(255) references Species(topiaId); +alter table SampleRow add column sclerochronologySamplingContext character varying(255); +alter table SampleRow add column individualMeasurementStrategy character varying(255); +alter table SampleRow add column sclerochronologySamplingContextInfo text; diff --git a/wao-persistence/src/main/resources/db/migration/V5_0__sclerochronology.sql b/wao-persistence/src/main/resources/db/migration/V5_0__sclerochronology.sql deleted file mode 100644 index fe74d39..0000000 --- a/wao-persistence/src/main/resources/db/migration/V5_0__sclerochronology.sql +++ /dev/null @@ -1,18 +0,0 @@ - - --- Ouverture de l'accès au programme sclérochronologie du compte admin -insert into UserProfile values ( - 'fr.ifremer.wao.entities.UserProfile#129111142543000#933200987654321550', - 1, - now(), - (select topiaId from WaoUser where login = 'admin'), - 'SCLEROCHRONOLOGY', - 'ADMIN', - true, - uuid_in(md5(random()::text || now()::text)::cstring) -); - -alter table SampleRow add column species character varying(255) references Species(topiaId); -alter table SampleRow add column sclerochronologySamplingContext character varying(255); -alter table SampleRow add column individualMeasurementStrategy character varying(255); -alter table SampleRow add column sclerochronologySamplingContextInfo text; -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/lucene in repository wao. See http://git.codelutin.com/wao.git commit d2305ae7858bd60907534017231e93f20eeffffd Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:05:13 2014 +0100 Correction d'une clause catch pour un import CSV --- .../wao/services/service/administration/ReferentialService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wao-services/src/main/java/fr/ifremer/wao/services/service/administration/ReferentialService.java b/wao-services/src/main/java/fr/ifremer/wao/services/service/administration/ReferentialService.java index c7e6880..01dee68 100644 --- a/wao-services/src/main/java/fr/ifremer/wao/services/service/administration/ReferentialService.java +++ b/wao-services/src/main/java/fr/ifremer/wao/services/service/administration/ReferentialService.java @@ -81,7 +81,6 @@ import org.nuiton.i18n.I18n; import org.nuiton.topia.persistence.TopiaDao; import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.topia.persistence.TopiaNoResultException; -import org.nuiton.topia.persistence.TopiaNonUniqueResultException; import org.nuiton.util.beans.Binder; import org.nuiton.util.beans.BinderFactory; @@ -822,7 +821,7 @@ public class ReferentialService extends WaoServiceSupport { } } - } catch (TopiaNonUniqueResultException e) { + } catch (ImportRuntimeException e) { throw new ImportErrorException(e); } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/lucene in repository wao. See http://git.codelutin.com/wao.git commit 0e7cf1248e9b2d4c021834d779c93701c55c8292 Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:05:35 2014 +0100 Correction d'un WARN struts en ajoutant un champs manquant dans l'action --- .../java/fr/ifremer/wao/web/action/AbstractImportCsvAction.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wao-web/src/main/java/fr/ifremer/wao/web/action/AbstractImportCsvAction.java b/wao-web/src/main/java/fr/ifremer/wao/web/action/AbstractImportCsvAction.java index 99168fa..df70a22 100644 --- a/wao-web/src/main/java/fr/ifremer/wao/web/action/AbstractImportCsvAction.java +++ b/wao-web/src/main/java/fr/ifremer/wao/web/action/AbstractImportCsvAction.java @@ -50,6 +50,8 @@ public abstract class AbstractImportCsvAction extends WaoJspActionSupport { protected String csvFileContentType; + protected String csvFileFileName; + public void setCsvFile(File csvFile) { this.csvFile = csvFile; } @@ -58,6 +60,10 @@ public abstract class AbstractImportCsvAction extends WaoJspActionSupport { this.csvFileContentType = csvFileContentType; } + public void setCsvFileFileName(String csvFileFileName) { + this.csvFileFileName = csvFileFileName; + } + public void setService(ReferentialService service) { this.service = service; } @@ -80,7 +86,7 @@ public abstract class AbstractImportCsvAction extends WaoJspActionSupport { csvInputStream = new FileInputStream(csvFile); if (log.isInfoEnabled()) { - log.info(getAuthenticatedWaoUser().getWaoUser() + " is doing a csv import in action " + this); + log.info(getAuthenticatedWaoUser().getWaoUser() + " is doing a csv import in action " + this + " with file " + csvFileFileName); } importCsv(csvInputStream); -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/lucene in repository wao. See http://git.codelutin.com/wao.git commit edbe89dd199254a0d764ebb170c69fe7a6e6aa5f Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:37:11 2014 +0100 Reprise des tests lucene de la banche lucene sur SVN --- pom.xml | 21 +++ wao-services/pom.xml | 17 +++ .../service/ObsMerSamplingPlanService.java | 5 + .../service/ObsMerSamplingPlanServiceTest.java | 147 +++++++++++++++++++++ 4 files changed, 190 insertions(+) diff --git a/pom.xml b/pom.xml index bca0260..0b74d5e 100644 --- a/pom.xml +++ b/pom.xml @@ -403,6 +403,27 @@ </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>4.9.0</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-facet</artifactId> + <version>4.9.0</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + <version>4.9.0</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>4.9.0</version> + </dependency> + + <dependency> <groupId>org.apache.struts.xwork</groupId> <artifactId>xwork-core</artifactId> <version>${struts2Version}</version> diff --git a/wao-services/pom.xml b/wao-services/pom.xml index 3d39d69..361b849 100644 --- a/wao-services/pom.xml +++ b/wao-services/pom.xml @@ -137,6 +137,23 @@ <artifactId>mockito-core</artifactId> </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-facet</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + </dependency> + </dependencies> <build> diff --git a/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java b/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java index 8d447f7..aa0d01e 100644 --- a/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java +++ b/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java @@ -39,6 +39,11 @@ import fr.ifremer.wao.services.service.csv.ObsMerSamplingPlanExportModel; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.util.Version; import org.nuiton.csv.Export; import org.nuiton.csv.ExportModel; diff --git a/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java b/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java index 76f8f4f..d737ac5 100644 --- a/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java +++ b/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java @@ -23,6 +23,7 @@ package fr.ifremer.wao.services.service; import com.google.common.collect.ImmutableSet; import fr.ifremer.wao.SampleRowsFilter; +import fr.ifremer.wao.entity.FishingZone; import fr.ifremer.wao.entity.SampleMonth; import fr.ifremer.wao.entity.SampleRow; import fr.ifremer.wao.services.AbstractWaoServiceTest; @@ -30,15 +31,44 @@ import fr.ifremer.wao.services.FakeWaoServiceContext; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.facet.DrillDownQuery; +import org.apache.lucene.facet.FacetField; +import org.apache.lucene.facet.FacetResult; +import org.apache.lucene.facet.Facets; +import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsConfig; +import org.apache.lucene.facet.LabelAndValue; +import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.util.Version; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.util.DateUtil; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import static fr.ifremer.wao.services.service.ObsMerSamplingPlan.ObsMerSamplingPlanSampleRowPart; @@ -191,4 +221,121 @@ public class ObsMerSamplingPlanServiceTest extends AbstractWaoServiceTest { Assert.assertEquals(oldMonth.getEstimated(), estimatedTidesValue, 0.01); } } + + @Test + public void testLucene() throws IOException, ParseException { + + testImportSamplingPlan(); + + Analyzer analyser = new StandardAnalyzer(Version.LUCENE_4_9); + + File luceneIndexPath = new File("/tmp/wao/lucene/index"); + Directory luceneIndexDirectory = new NIOFSDirectory(luceneIndexPath); + + File luceneTaxonomyPath = new File("/tmp/wao/lucene/taxonomy"); + Directory luceneTaxonomyDirectory = new NIOFSDirectory(luceneTaxonomyPath); + + IndexWriter writer = new IndexWriter(luceneIndexDirectory, new IndexWriterConfig(Version.LUCENE_4_9, analyser)); + TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(luceneTaxonomyDirectory, IndexWriterConfig.OpenMode.CREATE); + FacetsConfig config = new FacetsConfig(); + config.setMultiValued(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_FACADE_NAME, true); + config.setMultiValued(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_SECTOR_NAME, true); + + // pour vider l'index + writer.deleteAll(); + + List<SampleRow> sampleRows = serviceContext.getPersistenceContext().getSampleRowDao().findAll(); + + for (SampleRow sampleRow : sampleRows) { + + Document sampleRowDocument = new Document(); + sampleRowDocument.add(new TextField(TopiaEntity.PROPERTY_TOPIA_ID, sampleRow.getCode(), Field.Store.YES)); + + for (FishingZone fishingZone : sampleRow.getFishingZone()) { + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_FACADE_NAME, fishingZone.getFacadeName())); + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_SECTOR_NAME, fishingZone.getSectorName())); + } + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_CODE, sampleRow.getCode())); + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_PROGRAM_NAME, sampleRow.getProgramName())); + +// sampleRowCodes.add(FilterOption.forString(sampleRow.getCode())); +// programNames.add(FilterOption.forString(sampleRow.getProgramName())); +// +// Company company = sampleRow.getCompany(); +// if (company != null) { +// companies.add(FilterOption.forValueAndLabel(company.getTopiaId(), company.getName())); +// } +// samplingStrategies.add(FilterOption.forEnum(locale, sampleRow.getSamplingStrategy())); +// if (sampleRow.getObsProgram().isObsVente()) { +// TerrestrialLocation terrestrialDistrict = sampleRow.getTerrestrialLocation(); +// terrestrialDistricts.add( +// FilterOption.forValueAndLabel( +// terrestrialDistrict.getTopiaId(), +// terrestrialDistrict.getDescription()) +// ); +// } +// for (DCF5Code dcf5Code : sampleRow.getdCF5Code()) { +// FishingGearDCF fishingGearDCF = dcf5Code.getFishingGearDCF(); +// String fishingGearLabel = fishingGearDCF.getCode() + " - " + l(locale, fishingGearDCF.getI18nKey()); +// fishingGearDcfs.add( +// FilterOption.forValueAndLabel( +// fishingGearDCF.getTopiaId(), +// fishingGearLabel) +// ); +// TargetSpeciesDCF targetSpeciesDCF = dcf5Code.getTargetSpeciesDCF(); +// if (targetSpeciesDCF != null) { +// String targetspeciesLabel = targetSpeciesDCF.getCode() + " - " + l(locale, targetSpeciesDCF.getI18nKey()); +// targetSpeciesDcfs.add( +// FilterOption.forValueAndLabel( +// targetSpeciesDCF.getTopiaId(), +// targetspeciesLabel) +// ); +// } +// } + + writer.addDocument(config.build(taxonomyWriter, sampleRowDocument)); + + } + + writer.close(); + taxonomyWriter.close(); + + + + // opening a lucene index + IndexReader indexReader = DirectoryReader.open(luceneIndexDirectory); + IndexSearcher searcher = new IndexSearcher(indexReader); + TaxonomyReader taxoReader = new DirectoryTaxonomyReader(luceneTaxonomyDirectory); + + DrillDownQuery q = new DrillDownQuery(config); + q.add("programName", "PPP2010"); + + // create query +// QueryParser parser = new QueryParser(Version.LUCENE_4_9, "name", analyser); +// Query q = parser.parse("programName:PPP2010"); + +// Query q = new MatchAllDocsQuery(); + + + // perform search + FacetsCollector facetsCollector = new FacetsCollector(); + //FacetsCollector.search(searcher, q, 10, facetsCollector); + searcher.search(q, null, facetsCollector); + //TopScoreDocCollector topDocsCollector = TopScoreDocCollector.create(10, true); + //searcher.search(q, MultiCollector.wrap(topDocsCollector, facetsCollector)); + + Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, facetsCollector); + for (FacetResult facetResult : facets.getAllDims(10)) { + System.out.println("Facet : " + facetResult.dim + + " (total " + facetResult.childCount + ")"); + for (LabelAndValue labelAndValue : facetResult.labelValues) { + System.out.println(" - " + labelAndValue.label + " : " + labelAndValue.value); + } + } + + taxoReader.close(); + //searcher. + indexReader.close(); + + } } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm