This is an automated email from the git hooks/post-receive script. New commit to branch feature/lucene in repository wao. See http://git.codelutin.com/wao.git commit edbe89dd199254a0d764ebb170c69fe7a6e6aa5f Author: Brendan Le Ny <bleny@codelutin.com> Date: Mon Dec 22 15:37:11 2014 +0100 Reprise des tests lucene de la banche lucene sur SVN --- pom.xml | 21 +++ wao-services/pom.xml | 17 +++ .../service/ObsMerSamplingPlanService.java | 5 + .../service/ObsMerSamplingPlanServiceTest.java | 147 +++++++++++++++++++++ 4 files changed, 190 insertions(+) diff --git a/pom.xml b/pom.xml index bca0260..0b74d5e 100644 --- a/pom.xml +++ b/pom.xml @@ -403,6 +403,27 @@ </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>4.9.0</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-facet</artifactId> + <version>4.9.0</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + <version>4.9.0</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>4.9.0</version> + </dependency> + + <dependency> <groupId>org.apache.struts.xwork</groupId> <artifactId>xwork-core</artifactId> <version>${struts2Version}</version> diff --git a/wao-services/pom.xml b/wao-services/pom.xml index 3d39d69..361b849 100644 --- a/wao-services/pom.xml +++ b/wao-services/pom.xml @@ -137,6 +137,23 @@ <artifactId>mockito-core</artifactId> </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-facet</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + </dependency> + </dependencies> <build> diff --git a/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java b/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java index 8d447f7..aa0d01e 100644 --- a/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java +++ b/wao-services/src/main/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanService.java @@ -39,6 +39,11 @@ import fr.ifremer.wao.services.service.csv.ObsMerSamplingPlanExportModel; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.util.Version; import org.nuiton.csv.Export; import org.nuiton.csv.ExportModel; diff --git a/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java b/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java index 76f8f4f..d737ac5 100644 --- a/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java +++ b/wao-services/src/test/java/fr/ifremer/wao/services/service/ObsMerSamplingPlanServiceTest.java @@ -23,6 +23,7 @@ package fr.ifremer.wao.services.service; import com.google.common.collect.ImmutableSet; import fr.ifremer.wao.SampleRowsFilter; +import fr.ifremer.wao.entity.FishingZone; import fr.ifremer.wao.entity.SampleMonth; import fr.ifremer.wao.entity.SampleRow; import fr.ifremer.wao.services.AbstractWaoServiceTest; @@ -30,15 +31,44 @@ import fr.ifremer.wao.services.FakeWaoServiceContext; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.facet.DrillDownQuery; +import org.apache.lucene.facet.FacetField; +import org.apache.lucene.facet.FacetResult; +import org.apache.lucene.facet.Facets; +import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsConfig; +import org.apache.lucene.facet.LabelAndValue; +import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.util.Version; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.util.DateUtil; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import static fr.ifremer.wao.services.service.ObsMerSamplingPlan.ObsMerSamplingPlanSampleRowPart; @@ -191,4 +221,121 @@ public class ObsMerSamplingPlanServiceTest extends AbstractWaoServiceTest { Assert.assertEquals(oldMonth.getEstimated(), estimatedTidesValue, 0.01); } } + + @Test + public void testLucene() throws IOException, ParseException { + + testImportSamplingPlan(); + + Analyzer analyser = new StandardAnalyzer(Version.LUCENE_4_9); + + File luceneIndexPath = new File("/tmp/wao/lucene/index"); + Directory luceneIndexDirectory = new NIOFSDirectory(luceneIndexPath); + + File luceneTaxonomyPath = new File("/tmp/wao/lucene/taxonomy"); + Directory luceneTaxonomyDirectory = new NIOFSDirectory(luceneTaxonomyPath); + + IndexWriter writer = new IndexWriter(luceneIndexDirectory, new IndexWriterConfig(Version.LUCENE_4_9, analyser)); + TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(luceneTaxonomyDirectory, IndexWriterConfig.OpenMode.CREATE); + FacetsConfig config = new FacetsConfig(); + config.setMultiValued(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_FACADE_NAME, true); + config.setMultiValued(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_SECTOR_NAME, true); + + // pour vider l'index + writer.deleteAll(); + + List<SampleRow> sampleRows = serviceContext.getPersistenceContext().getSampleRowDao().findAll(); + + for (SampleRow sampleRow : sampleRows) { + + Document sampleRowDocument = new Document(); + sampleRowDocument.add(new TextField(TopiaEntity.PROPERTY_TOPIA_ID, sampleRow.getCode(), Field.Store.YES)); + + for (FishingZone fishingZone : sampleRow.getFishingZone()) { + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_FACADE_NAME, fishingZone.getFacadeName())); + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_FISHING_ZONE + "." + FishingZone.PROPERTY_SECTOR_NAME, fishingZone.getSectorName())); + } + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_CODE, sampleRow.getCode())); + sampleRowDocument.add(new FacetField(SampleRow.PROPERTY_PROGRAM_NAME, sampleRow.getProgramName())); + +// sampleRowCodes.add(FilterOption.forString(sampleRow.getCode())); +// programNames.add(FilterOption.forString(sampleRow.getProgramName())); +// +// Company company = sampleRow.getCompany(); +// if (company != null) { +// companies.add(FilterOption.forValueAndLabel(company.getTopiaId(), company.getName())); +// } +// samplingStrategies.add(FilterOption.forEnum(locale, sampleRow.getSamplingStrategy())); +// if (sampleRow.getObsProgram().isObsVente()) { +// TerrestrialLocation terrestrialDistrict = sampleRow.getTerrestrialLocation(); +// terrestrialDistricts.add( +// FilterOption.forValueAndLabel( +// terrestrialDistrict.getTopiaId(), +// terrestrialDistrict.getDescription()) +// ); +// } +// for (DCF5Code dcf5Code : sampleRow.getdCF5Code()) { +// FishingGearDCF fishingGearDCF = dcf5Code.getFishingGearDCF(); +// String fishingGearLabel = fishingGearDCF.getCode() + " - " + l(locale, fishingGearDCF.getI18nKey()); +// fishingGearDcfs.add( +// FilterOption.forValueAndLabel( +// fishingGearDCF.getTopiaId(), +// fishingGearLabel) +// ); +// TargetSpeciesDCF targetSpeciesDCF = dcf5Code.getTargetSpeciesDCF(); +// if (targetSpeciesDCF != null) { +// String targetspeciesLabel = targetSpeciesDCF.getCode() + " - " + l(locale, targetSpeciesDCF.getI18nKey()); +// targetSpeciesDcfs.add( +// FilterOption.forValueAndLabel( +// targetSpeciesDCF.getTopiaId(), +// targetspeciesLabel) +// ); +// } +// } + + writer.addDocument(config.build(taxonomyWriter, sampleRowDocument)); + + } + + writer.close(); + taxonomyWriter.close(); + + + + // opening a lucene index + IndexReader indexReader = DirectoryReader.open(luceneIndexDirectory); + IndexSearcher searcher = new IndexSearcher(indexReader); + TaxonomyReader taxoReader = new DirectoryTaxonomyReader(luceneTaxonomyDirectory); + + DrillDownQuery q = new DrillDownQuery(config); + q.add("programName", "PPP2010"); + + // create query +// QueryParser parser = new QueryParser(Version.LUCENE_4_9, "name", analyser); +// Query q = parser.parse("programName:PPP2010"); + +// Query q = new MatchAllDocsQuery(); + + + // perform search + FacetsCollector facetsCollector = new FacetsCollector(); + //FacetsCollector.search(searcher, q, 10, facetsCollector); + searcher.search(q, null, facetsCollector); + //TopScoreDocCollector topDocsCollector = TopScoreDocCollector.create(10, true); + //searcher.search(q, MultiCollector.wrap(topDocsCollector, facetsCollector)); + + Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, facetsCollector); + for (FacetResult facetResult : facets.getAllDims(10)) { + System.out.println("Facet : " + facetResult.dim + + " (total " + facetResult.childCount + ")"); + for (LabelAndValue labelAndValue : facetResult.labelValues) { + System.out.println(" - " + labelAndValue.label + " : " + labelAndValue.value); + } + } + + taxoReader.close(); + //searcher. + indexReader.close(); + + } } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.