Author: bpoussin Date: 2012-02-01 19:49:01 +0100 (Wed, 01 Feb 2012) New Revision: 1408 Url: http://nuiton.org/repositories/revision/wikitty/1408 Log: Evolution #1932: (Not)Equals must be ignore case and ignore accent if needed Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryMaker.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryParser.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorToString.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/Equals.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/NotEquals.java trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittyQueryVisitorToSolr.java Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryMaker.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryMaker.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryMaker.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -456,6 +456,39 @@ } /** + * @see {@link Equals} + */ + public WikittyQueryMaker eqIgnoreCaseAndAccent(Element element) { + addCondition(new Equals(element, true)); + return this; + } + + /** + * Equals. + * + * Restrict search so that the field value equals the parameter. + * + * You might use patterns in your equality. + * + * @param element the field on which the search is made + * @param value the value the element must be equals to + * @return {@code this} + * @see {@link Equals} + */ + public WikittyQueryMaker eqIgnoreCaseAndAccent(String fqfield, Object value) { + return eqIgnoreCaseAndAccent(new ElementField(fqfield), value); + } + + /** + * @see {@link Equals} + */ + public WikittyQueryMaker eqIgnoreCaseAndAccent(Element element, Object value) { + ConditionValue s = convertToConditionValue(value); + addCondition(new Equals(element, s, true)); + return this; + } + + /** * Extension equals. * * Restrict search to wikitties that got the extension in parameter. @@ -537,6 +570,38 @@ } /** + * @see {@link NotEquals} + */ + public WikittyQueryMaker neIgnoreCaseAndAccent(Element element) { + addCondition(new NotEquals(element, true)); + return this; + } + + /** + * Not equals. + * + * Restrict search to elements that are not equals to the value given in + * parameter. + * + * @param fqfield the element on which the restriction is put + * @param value the value the element must not be equals to. + * @return {@code this} with the {@code neq} restriction added. + * @see {@link NotEquals} + */ + public WikittyQueryMaker neIgnoreCaseAndAccent(String fqfield, Object value) { + return neIgnoreCaseAndAccent(new ElementField(fqfield), value); + } + + /** + * @see {@link NotEquals} + */ + public WikittyQueryMaker neIgnoreCaseAndAccent(Element element, Object value) { + ConditionValue s = convertToConditionValue(value); + addCondition(new NotEquals(element, s, true)); + return this; + } + + /** * Extension not equals. * * Restrict search to wikitties that do not get the extension given in Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryParser.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryParser.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryParser.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -106,6 +106,7 @@ public static final String CURLY_BRACKET_CLOSE = "}"; public static final String CURLY_BRACKET_OPEN = "{"; public static final String EQUALS = "="; + public static final String EQUALS_IGNORE_CASE_AND_ACCENT = "~"; public static final String GREATER = ">"; public static final String GREATER_OR_EQUALS = ">="; public static final String LESS = "<"; @@ -113,6 +114,7 @@ public static final String LIKE = "LIKE"; public static final String NOT = "NOT"; public static final String NOT_EQUALS = "!="; + public static final String NOT_EQUALS_IGNORE_CASE_AND_ACCENT = "!~"; public static final String OR = "OR"; public static final String BRACKET_CLOSE = ")"; public static final String BRACKET_OPEN = "("; @@ -291,7 +293,8 @@ not(), isNull(), isNotNull(), select(), greatereq(), lesseq(), between(), containsAll(), containsOne(), - eq(), neq(), less(), greater(), like(), notlike(), + eq(), neq(), eqIgnoreCaseAndAccent(), neqIgnoreCaseAndAccent(), + less(), greater(), like(), notlike(), rTrue(), rFalse(), keyword() ); } @@ -337,6 +340,24 @@ push(new NotEquals(toElement(pop(1).toString()), (ConditionValue)pop()))); } + /** + * gere eq, startsWith, endsWith, isNull + * @return + */ + Rule eqIgnoreCaseAndAccent() { + return Sequence(field(), push(match()), space(), EQUALS_IGNORE_CASE_AND_ACCENT, space(), value(), + push(new Equals(toElement(pop(1).toString()), (ConditionValue)pop(), true))); + } + + /** + * gere eq, isNull + * @return + */ + Rule neqIgnoreCaseAndAccent() { + return Sequence(field(), push(match()), space(), NOT_EQUALS_IGNORE_CASE_AND_ACCENT, space(), value(), + push(new NotEquals(toElement(pop(1).toString()), (ConditionValue)pop(), true))); + } + Rule less() { return Sequence(field(), push(match()), space(), LESS, space(), value(), push(new Less(toElement(pop(1).toString()), (ConditionValue)pop()))); Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorCopy.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -226,7 +226,11 @@ @Override public boolean visitEnter(Equals o) { - getQueryMaker().eq(o.getElement()); + if (o.isIgnoreCaseAndAccent()) { + getQueryMaker().eqIgnoreCaseAndAccent(o.getElement()); + } else { + getQueryMaker().eq(o.getElement()); + } return true; } @@ -237,7 +241,12 @@ @Override public boolean visitEnter(NotEquals o) { - getQueryMaker().ne(o.getElement()); + if (o.isIgnoreCaseAndAccent()) { + getQueryMaker().neIgnoreCaseAndAccent(o.getElement()); + } else { + getQueryMaker().ne(o.getElement()); + } + return true; } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorToString.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorToString.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/WikittyQueryVisitorToString.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -202,8 +202,12 @@ @Override public boolean visitEnter(Equals o) { - text += o.getElement().getValue() - + WikittyQueryParser.EQUALS; + text += o.getElement().getValue(); + if (o.isIgnoreCaseAndAccent()) { + text += WikittyQueryParser.EQUALS_IGNORE_CASE_AND_ACCENT; + } else { + text += WikittyQueryParser.EQUALS; + } return true; } @@ -302,8 +306,12 @@ @Override public boolean visitEnter(NotEquals o) { - text += o.getElement().getValue() - + WikittyQueryParser.NOT_EQUALS; + text += o.getElement().getValue(); + if (o.isIgnoreCaseAndAccent()) { + text += WikittyQueryParser.NOT_EQUALS_IGNORE_CASE_AND_ACCENT; + } else { + text += WikittyQueryParser.NOT_EQUALS; + } return true; } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/Equals.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/Equals.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/Equals.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -2,8 +2,8 @@ * #%L * Wikitty :: api * - * $Id: Equals.java 1136 2011-08-12 14:24:03Z tchemit $ - * $HeadURL: http://svn.nuiton.org/svn/wikitty/trunk/wikitty-api/src/main/java/org.nuiton... $ + * $Id$ + * $HeadURL$ * %% * Copyright (C) 2009 - 2010 CodeLutin, Benjamin Poussin * %% @@ -38,6 +38,10 @@ * * <li> {@link WikittyQueryMaker}.eq("myext.myfield", "*jour") match field "bonjour" but not "BONJOUR" * + * <p> + * You can specify ignoreCaseAndAccent to check equality. This mode + * ignore case and accent. example: "çéçù" == "CecU" in this mode. + * * @author poussin * @version $Revision$ * @since 3.3 @@ -50,6 +54,8 @@ // serialVersionUID is used for serialization. private static final long serialVersionUID = 1L; + protected boolean ignoreCaseAndAccent = false; + public Equals(Element element) { super(element); } @@ -61,4 +67,24 @@ public Equals(Element element, ConditionValue value) { super(element, value); } + + public Equals(Element element, boolean ignoreCaseAndAccent) { + super(element); + this.ignoreCaseAndAccent = ignoreCaseAndAccent; + } + + public Equals(Element element, String value, boolean ignoreCaseAndAccent) { + super(element, value); + this.ignoreCaseAndAccent = ignoreCaseAndAccent; + } + + public Equals(Element element, ConditionValue value, boolean ignoreCaseAndAccent) { + super(element, value); + this.ignoreCaseAndAccent = ignoreCaseAndAccent; + } + + public boolean isIgnoreCaseAndAccent() { + return ignoreCaseAndAccent; + } + } Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/NotEquals.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/NotEquals.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/query/conditions/NotEquals.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -39,6 +39,12 @@ * * <li> {@link WikittyQueryMaker}.ne("myext.myfield", "*jour") not match field "bonjour" but match "BONJOUR" * + * <p> + * You can specify ignoreCaseAndAccent to check equality. This mode + * ignore case and accent. + * + * <li> {@link WikittyQueryMaker}.neIgnoreCaseAndAccent("myext.myfield", "*jour") not match field "bonjour" and not match "BONJOUR" + * * @author poussin * @version $Revision$ * @since 3.3 @@ -51,6 +57,8 @@ // serialVersionUID is used for serialization. private static final long serialVersionUID = 1L; + protected boolean ignoreCaseAndAccent = false; + public NotEquals(Element element) { super(element); } @@ -63,4 +71,23 @@ super(element, value); } + public NotEquals(Element element, boolean ignoreCaseAndAccent) { + super(element); + this.ignoreCaseAndAccent = ignoreCaseAndAccent; + } + + public NotEquals(Element element, String value, boolean ignoreCaseAndAccent) { + super(element, value); + this.ignoreCaseAndAccent = ignoreCaseAndAccent; + } + + public NotEquals(Element element, ConditionValue value, boolean ignoreCaseAndAccent) { + super(element, value); + this.ignoreCaseAndAccent = ignoreCaseAndAccent; + } + + public boolean isIgnoreCaseAndAccent() { + return ignoreCaseAndAccent; + } + } \ No newline at end of file Modified: trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java =================================================================== --- trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/main/java/org/nuiton/wikitty/storage/WikittySearchEngineInMemory.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -44,7 +44,7 @@ import org.apache.commons.collections.bag.HashBag; import org.apache.commons.collections.map.LazyMap; import org.apache.commons.lang.ClassUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.wikitty.WikittyException; @@ -409,6 +409,64 @@ } }; + static private Predicate EqualsIgnoreCaseAndAccentPredicate = new Predicate() { + public boolean check(FieldType type, Collection values, Collection expected) { + boolean result = false; + if (values != null && expected.size() > 0) { + // gestion des type STRING differement car il peut y avoir des '*' + if (type != null && type.getType() == WikittyTypes.STRING) { + Iterator i = expected.iterator(); + String exp = String.valueOf(i.next()); + for (Object fieldValue : values) { + String val = String.valueOf(fieldValue); + result = matchString(val, exp, true); + if (result) { + // si une des valeurs correspond, on retourne true + break; + } + } + } else { + expected = CollectionUtils.subtract(expected, values); + // si lorsqu'on retire tous les elements en commun avec la valeur + // du champs il ne reste plus rien, c'est que le equals est + // vrai + result = expected.isEmpty(); + } + } + return result; + + } + }; + + static private Predicate NotEqualsIgnoreCaseAndAccentPredicate = new Predicate() { + public boolean check(FieldType type, Collection values, Collection expected) { + boolean result = true; + if (values != null && expected.size() > 0) { + // gestion des type STRING differement car il peut y avoir des '*' + if (type != null && type.getType() == WikittyTypes.STRING) { + Iterator i = expected.iterator(); + String exp = String.valueOf(i.next()); + for (Object fieldValue : values) { + String val = String.valueOf(fieldValue); + result = !matchString(val, exp, true); + if (!result) { + // si une des valeurs correspond, on retourne true + break; + } + } + } else { + expected = CollectionUtils.subtract(expected, values); + // si lorsqu'on retire tous les elements en commun avec la valeur + // du champs il ne reste plus rien, c'est que le equals est + // vrai et donc le not equals est faux + result = !expected.isEmpty(); + } + } + return result; + + } + }; + static private Predicate GreaterPredicate = new Predicate() { public boolean check(FieldType type, Collection values, Collection expected) { boolean result = false; @@ -515,10 +573,9 @@ String exp = String.valueOf(i.next()); for (Object fieldValue : values) { String val = String.valueOf(fieldValue); - // TODO poussin 20111228 verifie que le containsIgnoreCase - // supprime les accents, si ce n'est pas le cas, il faut - // trouver comment faire - result = StringUtils.containsIgnoreCase(val, exp); + result = StringUtils.containsIgnoreCase( + StringUtils.stripAccents(val), + StringUtils.stripAccents(exp)); if (result) { // si une des valeurs correspond, on retourne true break; @@ -593,39 +650,43 @@ * @param s string * @param sub string to match with 's' parameter. This string can start * or and with star '*' - * @param ignoreCase if true match is done in ingore case mode + * @param ignoreCaseAndAccent if true match is done in ingore case mode * @return true if sub match s */ - static private boolean matchString(String s, String sub, boolean ignoreCase) { - // TODO poussin 20111228 verifie que le containsIgnoreCase - // supprime les accents, si ce n'est pas le cas, il faut - // trouver comment faire - + static private boolean matchString(String s, String sub, boolean ignoreCaseAndAccent) { boolean result = false; if (sub.startsWith("*") && sub.endsWith("*")) { sub = StringUtils.substring(sub, 1, -1); - if (ignoreCase) { - result = StringUtils.containsIgnoreCase(s, sub); + if (ignoreCaseAndAccent) { + result = StringUtils.containsIgnoreCase( + StringUtils.stripAccents(s), + StringUtils.stripAccents(sub)); } else { result = StringUtils.contains(s, sub); } } else if (sub.startsWith("*")) { sub = StringUtils.substring(sub, 1); - if (ignoreCase) { - result = StringUtils.endsWithIgnoreCase(s, sub); + if (ignoreCaseAndAccent) { + result = StringUtils.endsWithIgnoreCase( + StringUtils.stripAccents(s), + StringUtils.stripAccents(sub)); } else { result = StringUtils.endsWith(s, sub); } } else if (sub.endsWith("*")) { sub = StringUtils.substring(sub, 0, -1); - if (ignoreCase) { - result = StringUtils.startsWithIgnoreCase(s, sub); + if (ignoreCaseAndAccent) { + result = StringUtils.startsWithIgnoreCase( + StringUtils.stripAccents(s), + StringUtils.stripAccents(sub)); } else { result = StringUtils.startsWith(s, sub); } } else { - if (ignoreCase) { - result = StringUtils.equalsIgnoreCase(s, sub); + if (ignoreCaseAndAccent) { + result = StringUtils.equalsIgnoreCase( + StringUtils.stripAccents(s), + StringUtils.stripAccents(sub)); } else { result = StringUtils.equals(s, sub); } @@ -1011,8 +1072,13 @@ public boolean visitEnter(Equals o) { boolean result = false; - result = check(EqualsPredicate, o.getElement(), - evalConditionValue(o.getValue())); + if (o.isIgnoreCaseAndAccent()) { + result = check(EqualsIgnoreCaseAndAccentPredicate, o.getElement(), + evalConditionValue(o.getValue())); + } else { + result = check(EqualsPredicate, o.getElement(), + evalConditionValue(o.getValue())); + } evalStack.push(result); return false; @@ -1027,8 +1093,13 @@ public boolean visitEnter(NotEquals o) { boolean result = false; - result = check(NotEqualsPredicate, o.getElement(), - evalConditionValue(o.getValue())); + if (o.isIgnoreCaseAndAccent()) { + result = check(NotEqualsIgnoreCaseAndAccentPredicate, o.getElement(), + evalConditionValue(o.getValue())); + } else { + result = check(NotEqualsPredicate, o.getElement(), + evalConditionValue(o.getValue())); + } evalStack.push(result); return false; Modified: trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java =================================================================== --- trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-api/src/test/java/org/nuiton/wikitty/WikittyClientTest.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -250,7 +250,7 @@ w = wikittyClient.store(w); Assert.fail("not null contraint don't work on String"); } catch (WikittyException eee) { - eee.printStackTrace(); +// eee.printStackTrace(); // ok id must not be null } @@ -259,7 +259,7 @@ w = wikittyClient.store(w); Assert.fail("not null contraint don't work in Collection"); } catch (WikittyException eee) { - eee.printStackTrace(); +// eee.printStackTrace(); // ok id must not be null } @@ -1458,7 +1458,7 @@ WikittyQueryResult<Product> results3 = wikittyClient.findAllByQuery(Product.class, query3); Assert.assertEquals(1, results3.getTotalResult()); - // ??? + // ???13 WikittyQuery query4 = new WikittyQueryMaker().eq("*.name." + WikittyTypes.STRING, "Lanfeust").end(); WikittyQueryResult<Product> results4 = wikittyClient.findAllByQuery(Product.class, query4); Assert.assertEquals(1, results4.getTotalResult()); @@ -1639,7 +1639,8 @@ { // avec accent specifiquement sur le champs name mais sans accent pour la recherche // specifique à solr - WikittyQuery query = new WikittyQueryMaker().eq(WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME + "_c_t", "ceca").end(); + WikittyQuery query = new WikittyQueryMaker().eqIgnoreCaseAndAccent( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME, "ceca").end(); WikittyQueryResult<String> result = wikittyClient.findAllByQuery(query); Assert.assertEquals(1, result.size()); } @@ -1647,7 +1648,8 @@ // avec accent specifiquement sur le champs name // mais sans accent pour la recherche // et et majuscule - WikittyQuery query = new WikittyQueryMaker().eq(WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME + "_c_t", "CECA").end(); + WikittyQuery query = new WikittyQueryMaker().eqIgnoreCaseAndAccent( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME, "CECA").end(); WikittyQueryResult<String> result = wikittyClient.findAllByQuery(query); Assert.assertEquals(1, result.size()); } @@ -1655,7 +1657,8 @@ // avec accent specifiquement sur le champs name // mais sans accent pour la recherche // et et majuscule - WikittyQuery query = new WikittyQueryMaker().like(WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME, "ceca").end(); + WikittyQuery query = new WikittyQueryMaker().like( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME, "ceca").end(); WikittyQueryResult<String> result = wikittyClient.findAllByQuery(query); Assert.assertEquals(1, result.size()); } Modified: trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittyQueryVisitorToSolr.java =================================================================== --- trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittyQueryVisitorToSolr.java 2012-02-01 09:33:28 UTC (rev 1407) +++ trunk/wikitty-solr/src/main/java/org/nuiton/wikitty/storage/solr/WikittyQueryVisitorToSolr.java 2012-02-01 18:49:01 UTC (rev 1408) @@ -318,8 +318,16 @@ @Override public boolean visitEnter(Equals o) { - solrQuery += element2solr(o.getElement()) + String element2solr = element2solr(o.getElement()); + if (element2solr.endsWith(WikittySolrConstant.SUFFIX_STRING)) { // is string + if (o.isIgnoreCaseAndAccent()) { + element2solr += WikittySolrConstant.SUFFIX_STRING_LOWERCASE; + } + } + + solrQuery += element2solr + ":" + evalConditionValue(o.getValue()); + return false; } @@ -330,7 +338,14 @@ @Override public boolean visitEnter(NotEquals o) { - solrQuery += "-" + element2solr(o.getElement()) + String element2solr = element2solr(o.getElement()); + if (element2solr.endsWith(WikittySolrConstant.SUFFIX_STRING)) { // is string + if (o.isIgnoreCaseAndAccent()) { + element2solr += WikittySolrConstant.SUFFIX_STRING_LOWERCASE; + } + } + + solrQuery += "-" + element2solr + ":" + evalConditionValue(o.getValue()); return false; }