Author: athimel Date: 2012-04-16 15:06:33 +0200 (Mon, 16 Apr 2012) New Revision: 170 Url: http://chorem.org/repositories/revision/vradi/170 Log: Prepare standalone mode Added: trunk/vradi-services/src/main/java/org/chorem/vradi/VradiStandaloneMain.java trunk/vradi-services/src/main/java/org/chorem/vradi/demo/ trunk/vradi-services/src/main/java/org/chorem/vradi/demo/StandaloneDataService.java trunk/vradi-services/src/main/resources/ooo/ Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfiguration.java trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfigurationHelper.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/VradiStorageServiceImpl.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/BindingManager.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormManager.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormTypeManager.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/MailingManager.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/ThesaurusManager.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/AssignXmlStreamTasks.java trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/TasksManager.java trunk/vradi-services/src/main/resources/i18n/vradi-services_fr_FR.properties trunk/vradi-services/src/main/resources/log4j.properties trunk/vradi-services/src/main/resources/vradi-services.properties trunk/vradi-services/src/test/java/org/chorem/vradi/services/InternalUsageTest.java Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfiguration.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfiguration.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfiguration.java 2012-04-16 13:06:33 UTC (rev 170) @@ -196,6 +196,11 @@ _("vradi.service.config.smtp.port.description"), "25", Integer.class, false, false), + RECEIVE_MAIL_ENABLED( + "vradi.mail.receive.enabled", + _("vradi.service.config.mail.receive.enabled.description"), + "true", Boolean.class, false, false), + IMAP_HOST( "vradi.imap.host", _("vradi.service.config.imap.host.description"), Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfigurationHelper.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfigurationHelper.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/VradiServiceConfigurationHelper.java 2012-04-16 13:06:33 UTC (rev 170) @@ -286,4 +286,9 @@ int endpoint = config.getOptionAsInt(VradiServiceOption.SERVICE_PORT.key); return endpoint; } + + public static boolean isReceiveMailsEnabled(ApplicationConfig config) { + boolean result = config.getOptionAsBoolean(VradiServiceOption.RECEIVE_MAIL_ENABLED.key); + return result; + } } Added: trunk/vradi-services/src/main/java/org/chorem/vradi/VradiStandaloneMain.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/VradiStandaloneMain.java (rev 0) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/VradiStandaloneMain.java 2012-04-16 13:06:33 UTC (rev 170) @@ -0,0 +1,39 @@ +package org.chorem.vradi; + +import org.chorem.vradi.demo.StandaloneDataService; +import org.chorem.vradi.services.VradiException; +import org.chorem.vradi.services.VradiServiceContext; +import org.chorem.vradi.services.VradiStorageServiceImpl; +import org.chorem.vradi.services.managers.BindingManager; +import org.chorem.vradi.services.managers.FormManager; +import org.chorem.vradi.services.managers.FormTypeManager; +import org.chorem.vradi.services.managers.SearchManager; +import org.chorem.vradi.services.managers.ThesaurusManager; +import org.nuiton.util.ApplicationConfig; +import org.nuiton.wikitty.WikittyProxy; + +/** + * @author Arnaud Thimel <thimel@codelutin.com> + */ +public class VradiStandaloneMain { + + public static void main(String[] args) throws VradiException { + ApplicationConfig config = VradiServiceConfiguration.getConfig(); + WikittyProxy wikittyProxy = VradiServiceContext.getWikittyProxy(); + + FormTypeManager formTypeManager = new FormTypeManager(config, wikittyProxy); + ThesaurusManager thesaurusManager = new ThesaurusManager(wikittyProxy); + SearchManager searchManager = new SearchManager(wikittyProxy, thesaurusManager); + FormManager formManager = new FormManager(config, wikittyProxy, thesaurusManager, searchManager); + BindingManager bindingManager = new BindingManager(config, wikittyProxy, formTypeManager, formManager); + + StandaloneDataService standaloneDataService = new StandaloneDataService(config, bindingManager, formTypeManager, searchManager, thesaurusManager, wikittyProxy); + standaloneDataService.checkThesaurus(); + standaloneDataService.checkStreams(); + standaloneDataService.checkGroups(); + + VradiStorageServiceImpl vradiStorageService = new VradiStorageServiceImpl(); // Needed to start tasks programming + + } + +} Added: trunk/vradi-services/src/main/java/org/chorem/vradi/demo/StandaloneDataService.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/demo/StandaloneDataService.java (rev 0) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/demo/StandaloneDataService.java 2012-04-16 13:06:33 UTC (rev 170) @@ -0,0 +1,228 @@ +package org.chorem.vradi.demo; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.io.Files; +import com.google.common.io.InputSupplier; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.chorem.vradi.VradiConstants; +import org.chorem.vradi.VradiServiceConfigurationHelper; +import org.chorem.vradi.entities.FieldTypeEnum; +import org.chorem.vradi.entities.Form; +import org.chorem.vradi.entities.Group; +import org.chorem.vradi.entities.GroupImpl; +import org.chorem.vradi.entities.Infogene; +import org.chorem.vradi.entities.Query; +import org.chorem.vradi.entities.QueryImpl; +import org.chorem.vradi.entities.RootThesaurus; +import org.chorem.vradi.entities.Thesaurus; +import org.chorem.vradi.entities.ThesaurusImpl; +import org.chorem.vradi.entities.User; +import org.chorem.vradi.entities.UserImpl; +import org.chorem.vradi.entities.XmlFieldBinding; +import org.chorem.vradi.entities.XmlFieldBindingImpl; +import org.chorem.vradi.entities.XmlStream; +import org.chorem.vradi.entities.XmlStreamImpl; +import org.chorem.vradi.services.VradiException; +import org.chorem.vradi.services.managers.BindingManager; +import org.chorem.vradi.services.managers.FormTypeManager; +import org.chorem.vradi.services.managers.SearchManager; +import org.chorem.vradi.services.managers.ThesaurusManager; +import org.nuiton.util.ApplicationConfig; +import org.nuiton.util.DateUtil; +import org.nuiton.wikitty.WikittyProxy; +import org.nuiton.wikitty.entities.FieldType; +import org.nuiton.wikitty.entities.WikittyExtension; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @author Arnaud Thimel <thimel@codelutin.com> + */ +public class StandaloneDataService { + + private static final Log log = LogFactory.getLog(StandaloneDataService.class); + + protected ApplicationConfig config; + protected SearchManager searchManager; + protected BindingManager bindingManager; + protected FormTypeManager formTypeManager; + protected ThesaurusManager thesaurusManager; + protected WikittyProxy wikittyProxy; + + protected static final String APPEL_OFFRES_FORM_TYPE = "APPEL_OFFRES"; + protected static final String APPEL_OFFRES_TEMPLATE = "default_vradi_template.odt"; + + public StandaloneDataService(ApplicationConfig config, BindingManager bindingManager, FormTypeManager formTypeManager, SearchManager searchManager, ThesaurusManager thesaurusManager, WikittyProxy wikittyProxy) { + this.config = config; + this.bindingManager = bindingManager; + this.formTypeManager = formTypeManager; + this.searchManager = searchManager; + this.thesaurusManager = thesaurusManager; + this.wikittyProxy = wikittyProxy; + } + + protected WikittyExtension createAPPEL_OFFRESFormType() throws VradiException { + Map<String, FieldType> fields = new LinkedHashMap<String, FieldType>(); + fields.put("URL_AO", FieldTypeEnum.STRING.getFieldType()); + fields.put("Contact_technique", FieldTypeEnum.STRING.getFieldType()); + fields.put("Site_acheteur", FieldTypeEnum.STRING.getFieldType()); + fields.put("Contact_administratif", FieldTypeEnum.STRING.getFieldType()); + fields.put("category", FieldTypeEnum.STRING.getFieldType()); + fields.put("documents", FieldTypeEnum.STRING.getFieldType()); + + HashMap<String, String> tagValues = Maps.newHashMap(); + tagValues.put(VradiConstants.FormTypeTemplateEnum.FIELD_TEMPLATE_ATTRIBUTE.getValue(), APPEL_OFFRES_TEMPLATE); + WikittyExtension extension = formTypeManager.updateFormType(APPEL_OFFRES_FORM_TYPE, fields, Arrays.asList(Form.EXT_FORM), tagValues); + + + File templatesDir = VradiServiceConfigurationHelper.getTemplatesDir(config); + File templateDir = new File(templatesDir, APPEL_OFFRES_FORM_TYPE); + templateDir.mkdirs(); + File targetFile = new File(templateDir, APPEL_OFFRES_TEMPLATE); + + try { + Files.copy(new InputSupplier<InputStream>() { + @Override + public InputStream getInput() throws IOException { + return StandaloneDataService.class.getResourceAsStream("/ooo/default_vradi_template.odt"); + } + }, targetFile); + } catch (IOException eee) { + if (log.isWarnEnabled()) { + log.warn("Cannot write template file"); + } + } + + LinkedHashMap<String, String> outputMapping = Maps.newLinkedHashMap(); + outputMapping.put("objet", Infogene.FQ_FIELD_INFOGENE_OBJET); + outputMapping.put("date", Form.FQ_FIELD_FORM_DATEPUB); + outputMapping.put("id", Form.FQ_FIELD_INFOGENE_ID); + outputMapping.put("url", Infogene.FQ_FIELD_INFOGENE_SOURCEURL); + outputMapping.put("contact", APPEL_OFFRES_FORM_TYPE + ".Contact_administratif"); + outputMapping.put("department", Infogene.FQ_FIELD_INFOGENE_DEPARTMENT); + outputMapping.put("description", Infogene.FQ_FIELD_INFOGENE_DESCRIPTION); + formTypeManager.setAssociatedFields(APPEL_OFFRES_FORM_TYPE, APPEL_OFFRES_TEMPLATE, outputMapping); + return extension; + } + + protected XmlFieldBinding createXmlFieldBinding(String xmlField, String formField) throws VradiException { + XmlFieldBinding binding = new XmlFieldBindingImpl(); + binding.setXmlField(ImmutableSet.of(xmlField)); + binding.setFormField(formField); + + XmlFieldBinding result = wikittyProxy.store(binding); + return result; + } + + protected XmlStream createAPPEL_OFFRESXmlStream(String name, String url) throws VradiException { + XmlStream stream = new XmlStreamImpl(); + stream.setName(name); + stream.setUrl(url); + stream.setFormTypeName(APPEL_OFFRES_FORM_TYPE); + stream.setAutoAssign(true); + stream.setImportTime("08:00"); // TODO AThimel 16/04/2012 Move to conf + // tous + stream.addXmlFieldBinding(createXmlFieldBinding("title", Infogene.FQ_FIELD_INFOGENE_OBJET).getWikittyId()); + stream.addXmlFieldBinding(createXmlFieldBinding("link", Infogene.FQ_FIELD_INFOGENE_SOURCEURL).getWikittyId()); + stream.addXmlFieldBinding(createXmlFieldBinding("description", Infogene.FQ_FIELD_INFOGENE_DESCRIPTION).getWikittyId()); + stream.addXmlFieldBinding(createXmlFieldBinding("pubDate", Form.FQ_FIELD_FORM_DATEPUB).getWikittyId()); + + // marchésonline + stream.addXmlFieldBinding(createXmlFieldBinding("category", APPEL_OFFRES_FORM_TYPE + ".category").getWikittyId()); + stream.addXmlFieldBinding(createXmlFieldBinding("dc:creator", Infogene.FQ_FIELD_INFOGENE_ENTITY).getWikittyId()); + // EdiTender + stream.addXmlFieldBinding(createXmlFieldBinding("guid", APPEL_OFFRES_FORM_TYPE + ".documents").getWikittyId()); + // EdiTender et e-marchespublics + stream.addXmlFieldBinding(createXmlFieldBinding("author", APPEL_OFFRES_FORM_TYPE + ".Contact_administratif").getWikittyId()); + // e-marchespublics + stream.addXmlFieldBinding(createXmlFieldBinding("department", Infogene.FQ_FIELD_INFOGENE_DEPARTMENT).getWikittyId()); + wikittyProxy.store(stream); + + return stream; + } + + protected void checkStream(List<XmlStream> allStreams, String name, String url) throws VradiException { + boolean found = false; + for (XmlStream stream : allStreams) { + found |= name.equals(stream.getName()); + } + if (!found) { + createAPPEL_OFFRESXmlStream(name, url); + } + } + + public void checkStreams() throws VradiException { + + createAPPEL_OFFRESFormType(); + + List<XmlStream> allXmlStreams = bindingManager.getAllXmlStreams(); + + checkStream(allXmlStreams, "Marchés Online", "http://www.marchesonline.com/mol/rss/appels-d-offres-services-informatiques...."); + checkStream(allXmlStreams, "Edi Tender", "http://www.edi-tender.com/rss/marchespublics.xml"); + checkStream(allXmlStreams, "e-Marchés Publics", "https://www.e-marchespublics.com/rss/rss.php?v=2&l=200&c=Service"); + + } + public static final List<String> keywords = Lists.newArrayList( // TODO AThimel 16/04/2012 Move to conf + "Java", + "J2EE", "JEE", "EJB", + "swing", "struts", "hibernate", + "Tomcat", "Jetty", "jboss", "jonas", + "maven" + ); + + public void checkGroups() { + List<Group> groups = searchManager.findGroupsWithQueries(); + if (groups == null || groups.isEmpty()) { + + Query query = new QueryImpl(); // No need to specify query, return all + query.setName("Daily standalone request"); + wikittyProxy.store(query); + + User user = new UserImpl(); + user.setEmail("thimel@codelutin.com"); // TODO AThimel 16/04/2012 Move to conf + user.setName("Standalone user"); + user.setEnable(true); + user.setBeginSearchDate(DateUtil.createDate(1, 1, 2012)); + wikittyProxy.store(user); + + Group codeLutin = new GroupImpl(); + codeLutin.setName("Standalone group"); + codeLutin.setQueries(Sets.newHashSet(query.getWikittyId())); + codeLutin.setUser(Sets.newHashSet(user.getWikittyId())); + wikittyProxy.store(codeLutin); + } + + } + + protected void createThesaurus(RootThesaurus parent, String name) { + Thesaurus thesaurus = new ThesaurusImpl(); + thesaurus.setName(name); + thesaurus.setRootThesaurus(parent.getWikittyId()); + thesaurus.setParent(parent.getWikittyId()); + thesaurus.setTags(Sets.newHashSet(name)); + wikittyProxy.store(thesaurus); + } + + public void checkThesaurus() throws VradiException { + List<RootThesaurus> rootThesaurusList = thesaurusManager.getRootThesaurus(); + if (rootThesaurusList.isEmpty()) { + RootThesaurus rootThesaurus = thesaurusManager.createRootThesaurus("VradiRootThesaurus"); + + for (String keyword : keywords) { + createThesaurus(rootThesaurus, keyword); + } + } + + } +} Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/VradiStorageServiceImpl.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/VradiStorageServiceImpl.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/VradiStorageServiceImpl.java 2012-04-16 13:06:33 UTC (rev 170) @@ -562,6 +562,6 @@ @Override public Form updateForms(Form form, List<String> thesaurusIdsToAttach) throws VradiException { - return formManager.updateForm(form, thesaurusIdsToAttach); + return formManager.updateFormAndAttachToThesaurus(form, thesaurusIdsToAttach); } } Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/BindingManager.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/BindingManager.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/BindingManager.java 2012-04-16 13:06:33 UTC (rev 170) @@ -32,6 +32,7 @@ import org.chorem.vradi.beans.XmlStreamImportResult; import org.chorem.vradi.entities.Form; import org.chorem.vradi.entities.FormImpl; +import org.chorem.vradi.entities.ModificationTag; import org.chorem.vradi.entities.WebHarvestStream; import org.chorem.vradi.entities.XmlFieldBinding; import org.chorem.vradi.entities.XmlStream; @@ -621,6 +622,10 @@ form.setXmlStream(stream.getWikittyId()); form.setStatus(toTreatId); form.setImportContentHash(contentSHA1Hash); + + ModificationTag modificationTag = wikittyProxy.cast(form, ModificationTag.class); + modificationTag.setLastModified(new Date()); + forms.add(form); if (forms.size() > 100) { Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormManager.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormManager.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormManager.java 2012-04-16 13:06:33 UTC (rev 170) @@ -23,6 +23,11 @@ */ package org.chorem.vradi.services.managers; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -58,12 +63,14 @@ import org.nuiton.wikitty.search.Search; import org.nuiton.wikitty.search.operators.Element; +import javax.annotation.Nullable; import java.text.ParseException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -110,7 +117,7 @@ * @return forms saved * @throws VradiException if exception occured */ - public Form updateForm(Form form, Collection<String> thesaurus) throws VradiException { + public Form updateFormAndAttachToThesaurus(Form form, Collection<String> thesaurus) throws VradiException { return updateForms(Collections.singletonList(form), thesaurus).get(0); } @@ -408,16 +415,16 @@ } // Replace replace forms by previous - formsIdsToBind = repaceByPreviousReplacedForms(formsIdsToBind); + formsIdsToBind = replaceByPreviousReplacedForms(formsIdsToBind); ArrayList<String> formsIdsAsList = new ArrayList<String>(formsIdsToBind); Set<Form> formsToBind = new HashSet<Form>(wikittyProxy.restore(Form.class, formsIdsAsList)); // Remove already bound forms with queryMaker - formsToBind = removeAlreadyBound(currentSession, formsToBind, user); + removeAlreadyBound(currentSession, formsToBind, user); - formsToBind = removeNotConcernedForms(sending, user, formsToBind); + removeNotConcernedForms(sending, user, formsToBind); if (log.isDebugEnabled()) { log.debug("Adding " + formsToBind.size() + " forms to sending"); } @@ -498,7 +505,8 @@ // Limited on older inscription date Search filterDateModification = filter.or(); - Date olderInscriptionDate = getOlderInscriptionDate(group.getUser()); + Set<String> userIds = group.getUser(); + Date olderInscriptionDate = getOldestInscriptionDate(userIds); // Limited on date modification filterDateModification.isNull(ModificationTag.FQ_FIELD_MODIFICATIONTAG_LASTMODIFIED); @@ -520,6 +528,15 @@ // Extract result Set<String> resultFormsIds = new HashSet<String>(vradiFormPage.getFormsIdsToShow()); + Set<Form> forms = wikittyProxy.restore(Form.class, resultFormsIds); + for (Form form : forms) { + Status statusFound = wikittyProxy.restore(Status.class, form.getStatus()); + if (log.isWarnEnabled()) { + log.warn("form: " + form + "; status: " + statusFound); + } + + } + if (log.isDebugEnabled()) { log.debug("Query " + query + " return " + resultFormsIds.size() + " forms"); } @@ -529,19 +546,19 @@ throw new VradiException("Can't execute group query", eee); } catch (ParseException eee) { - throw new VradiException("Cant format date : ", eee); + throw new VradiException("Can't format date : ", eee); } } return formsIdsToBind; } /** - * Use to replace forms by previous replace forms + * Used to replace forms by previous replace forms * * @param resultFormsIds to inspect * @return return forms with previous replace forms */ - protected Set<String> repaceByPreviousReplacedForms(Set<String> resultFormsIds) { + protected Set<String> replaceByPreviousReplacedForms(Set<String> resultFormsIds) { List<String> copy = new ArrayList<String>(resultFormsIds); for (String formId : copy) { String previousReplaceFormId = getPreviousReplaceForm(formId); @@ -632,13 +649,15 @@ return wikittyProxy.restore(Sending.class, new ArrayList<String>(sendingsIds)); } - protected Set<Form> removeNotConcernedForms(Sending sending, User user, Set<Form> finalFormsToBind) { + protected void removeNotConcernedForms(Sending sending, User user, Set<Form> finalFormsToBind) { // Get user begin search date Date beginSearchDate = user.getBeginSearchDate(); // Check date - for (Form f : new ArrayList<Form>(finalFormsToBind)) { + Iterator<Form> iterator = finalFormsToBind.iterator(); + while (iterator.hasNext()) { + Form f = iterator.next(); // Using last modification date ModificationTag formTag = wikittyProxy.cast(f, ModificationTag.class); @@ -650,59 +669,48 @@ // If not enable, not attach forms if (!user.getEnable()) { if (log.isDebugEnabled()) { - log.debug("Removing form '" + - f.getObjet() + - "' because user (" + - user.getName() + - ") is not enable"); + String message = String.format("Removing form '%s' because user (%s) is not enabled", + f.getObjet(), user.getName()); + log.debug(message); } - finalFormsToBind.remove(f); + iterator.remove(); } // If date of peremption of form is pass if (lastModifier != null && lastModifier.before(beginSearchDate)) { if (log.isDebugEnabled()) { - log.debug("Removing form '" + - f.getObjet() + - "' because date of last modification (" + - peremptionDate + - ") is pass for user : " + - user.getName()); + String message = String.format("Removing form '%s' because date of last modification (%s) is past for user (%s)", + f.getObjet(), peremptionDate, user.getName()); + log.debug(message); } - finalFormsToBind.remove(f); + iterator.remove(); } // If date of peremption of form is pass if (peremptionDate != null && peremptionDate.before(new Date())) { if (log.isDebugEnabled()) { - log.debug("Removing form '" + - f.getObjet() + - "' because date of peremption (" + - peremptionDate + - ") is pass for user : " + - user.getName()); + String message = String.format("Removing form '%s' because date of last modification (%s) is past for user (%s)", + f.getObjet(), peremptionDate, user.getName()); + log.debug(message); } - finalFormsToBind.remove(f); + iterator.remove(); } // Remove deleted if are reassociated Set<String> deletedFormsIds = SendingHelper.extractForms(wikittyProxy, sending.getDeletedGroupForms()); String fid = f.getWikittyId(); if (deletedFormsIds.contains(fid)) { - finalFormsToBind.remove(f); if (log.isDebugEnabled()) { - log.debug("Removing form '" + - f.getObjet() + - "' because form is already deleted" + - " is pass for user : " + - user.getName()); + String message = String.format("Removing form '%s' because form is already deleted for user (%s)", + f.getObjet(), user.getName()); + log.debug(message); } + iterator.remove(); } } - return finalFormsToBind; } /** @@ -714,13 +722,16 @@ * @param user * @return */ - protected Set<Form> removeAlreadyBound(Session currentSession, Set<Form> forms, User user) { - Set<String> formIds = new HashSet<String>(); + protected void removeAlreadyBound(Session currentSession, Set<Form> forms, User user) { // Extract formIds - for (Form form : forms) { - formIds.add(form.getWikittyId()); - } + Iterable<String> transform = Iterables.transform(forms, new Function<Form, String>() { + @Override + public String apply(Form input) { + return input.getWikittyId(); + } + }); + Set<String> formIds = Sets.newLinkedHashSet(transform); // Recherche d'un sending existant String userId = user.getWikittyId(); @@ -728,7 +739,7 @@ // Search all groupForms concerned Search searchGroupForm = Search.query(); - // FIXME sletellier 24/10/10 : find a way to join tow requests + // FIXME sletellier 24/10/10 : find a way to join 2 requests searchGroupForm.contains(GroupForms.FQ_FIELD_GROUPFORMS_FORM, formIds); List<GroupForms> groupForms = wikittyProxy.findAllByCriteria(GroupForms.class, searchGroupForm.criteria()).getAll(); List<String> groupFormsId = new ArrayList<String>(); @@ -765,37 +776,46 @@ PagedResult<Sending> pagedResult = wikittyProxy.findAllByCriteria(Sending.class, criteria); + final Set<String> toRemoveIds = Sets.newHashSet(); for (Sending send : pagedResult.getAll()) { Set<String> formsIds = SendingHelper.extractForms(wikittyProxy, send.getGroupForms()); if (formsIds != null) { - formIds.removeAll(formsIds); + toRemoveIds.addAll(formsIds); } Set<String> deletedIds = SendingHelper.extractForms(wikittyProxy, send.getDeletedGroupForms()); if (deletedIds != null) { - formIds.removeAll(deletedIds); + toRemoveIds.addAll(deletedIds); } } + + Iterables.removeIf(forms, new Predicate<Form>() { + @Override + public boolean apply(@Nullable Form input) { + if (input == null) { + return true; + } + return toRemoveIds.contains(input.getWikittyId()); + } + }); + if (log.isDebugEnabled()) { - log.debug("Rest " + formIds.size() + " forms"); + log.debug(forms.size() + " forms remaining"); } - - List<Form> result = wikittyProxy.restore(Form.class, new ArrayList<String>(formIds)); - return new HashSet<Form>(result); } /** * Retourne la plus ancienne date du query maker si c'est une date - * ou des user gu groupe si c'est un groupe. + * ou des user du groupe si c'est un groupe. * * @param usersIds * @return olderInscriptionDate */ - protected Date getOlderInscriptionDate(Set<String> usersIds) { + protected Date getOldestInscriptionDate(Set<String> usersIds) { Date result = new Date(); if (usersIds != null) { - for (String userId : usersIds) { - User user = wikittyProxy.restore(User.class, userId); + Set<User> users = wikittyProxy.restore(User.class, usersIds); + for (User user : users) { Date beginSearchDate = user.getBeginSearchDate(); if (beginSearchDate != null && result.after(DateUtil.setMinTimeOfDay(beginSearchDate))) { @@ -1379,13 +1399,13 @@ log.debug("createDefaultStatuses()"); } - List<Status> all = new ArrayList<Status>(); - all.add(getNonTraiteStatus()); - all.add(getNonSelectionneStatus()); - all.add(getPreselectionneStatus()); - all.add(getSelectionneStatus()); + List<Status> all = Lists.newArrayList( + getNonTraiteStatus(), + getNonSelectionneStatus(), + getPreselectionneStatus(), + getSelectionneStatus()); - return new ArrayList<Status>(all); + return all; } public Status getNonTraiteStatus() { Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormTypeManager.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormTypeManager.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/FormTypeManager.java 2012-04-16 13:06:33 UTC (rev 170) @@ -231,8 +231,15 @@ return null; } File templatesDir = VradiServiceConfigurationHelper.getTemplatesDir(config); - File template = new File(new File(templatesDir, extensionName), templateName); - return template.exists() ? template : null; + File subDir = new File(templatesDir, extensionName); + File template = new File(subDir, templateName); + if (!template.exists()) { + if (log.isWarnEnabled()) { + log.warn("Requested template not found : " + template.getAbsolutePath()); + } + return null; + } + return template; } /** Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/MailingManager.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/MailingManager.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/MailingManager.java 2012-04-16 13:06:33 UTC (rev 170) @@ -789,7 +789,7 @@ } } else if (fieldBindings == null) { if (log.isWarnEnabled()) { - log.warn("No association field found, abording"); + log.warn("No association field found, aborting"); } } else { TemplateManager templateManager = new TemplateManager(config, template); Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/ThesaurusManager.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/ThesaurusManager.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/managers/ThesaurusManager.java 2012-04-16 13:06:33 UTC (rev 170) @@ -142,7 +142,7 @@ /** * Returns all children of the specified {@code thesaurusId} sorted by - * ordre and name. + * order and name. * * @param thesaurusId thesaurus wikitty id * @return all list of <code>TreeNodeImpl</code> Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/AssignXmlStreamTasks.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/AssignXmlStreamTasks.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/AssignXmlStreamTasks.java 2012-04-16 13:06:33 UTC (rev 170) @@ -86,7 +86,7 @@ List<Thesaurus> thesaurusProposed = thesaurusManager.proposeThesaurus(formId); if (!thesaurusProposed.isEmpty()) { Form form = wikittyProxy.restore(Form.class, formId); - form = formManager.updateForm(form, WikittyUtil.getIds(thesaurusProposed)); + form = formManager.updateFormAndAttachToThesaurus(form, WikittyUtil.getIds(thesaurusProposed)); // Set form as selected form.setStatus(formManager.getSelectionneStatus().getWikittyId()); Modified: trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/TasksManager.java =================================================================== --- trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/TasksManager.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/java/org/chorem/vradi/services/tasks/TasksManager.java 2012-04-16 13:06:33 UTC (rev 170) @@ -70,9 +70,6 @@ protected ApplicationConfig config; - /** Cron task to check mails every 10 minutes. */ - protected Timer mailCronTimer; - /** Cron task to import streams. */ protected ScheduledExecutorService xmlStreamScheduler; @@ -138,9 +135,11 @@ /** Init and start mail tasks (check every 10 minutes) */ protected void initMailTask() { - mailCronTimer = new Timer("vradi-mail-task"); - receiveMailTasks = new ReceiveMailTasks(mailingManager); - mailCronTimer.schedule(receiveMailTasks, 0, 10 * 60 * 1000); + if (VradiServiceConfigurationHelper.isReceiveMailsEnabled(config)) { + Timer mailCronTimer = new Timer("vradi-mail-task"); + receiveMailTasks = new ReceiveMailTasks(mailingManager); + mailCronTimer.schedule(receiveMailTasks, 0, 10 * 60 * 1000); + } } /** Init and start xml stream tasks. */ @@ -207,7 +206,7 @@ delayInMinute += 24 * 60; } if (log.isInfoEnabled()) { - log.info(task.getClass().getName() + " - in scheduler, delai : " + delayInMinute + "min, period " + 24 * 60 + "min"); + log.info(task.getClass().getName() + " - in scheduler [delay=" + delayInMinute + "min ; period=" + 24 * 60 + "min]"); } return scheduledExecutorService.scheduleAtFixedRate(task, delayInMinute, 24 * 60, TimeUnit.MINUTES); Modified: trunk/vradi-services/src/main/resources/i18n/vradi-services_fr_FR.properties =================================================================== --- trunk/vradi-services/src/main/resources/i18n/vradi-services_fr_FR.properties 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/resources/i18n/vradi-services_fr_FR.properties 2012-04-16 13:06:33 UTC (rev 170) @@ -48,6 +48,7 @@ vradi.service.config.mail.fromname.description= vradi.service.config.mail.offer.subject.description= vradi.service.config.mail.password.description= +vradi.service.config.mail.receive.enabled.description= vradi.service.config.mail.user.description= vradi.service.config.migration.skip.description= vradi.service.config.oOo.exec.folder.description= Modified: trunk/vradi-services/src/main/resources/log4j.properties =================================================================== --- trunk/vradi-services/src/main/resources/log4j.properties 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/resources/log4j.properties 2012-04-16 13:06:33 UTC (rev 170) @@ -23,7 +23,7 @@ ### # Global logging configuration -log4j.rootLogger=ERROR, stdout, file +log4j.rootLogger=WARN, stdout, file # Console output log4j.appender.stdout=org.apache.log4j.ConsoleAppender Modified: trunk/vradi-services/src/main/resources/vradi-services.properties =================================================================== --- trunk/vradi-services/src/main/resources/vradi-services.properties 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/main/resources/vradi-services.properties 2012-04-16 13:06:33 UTC (rev 170) @@ -46,7 +46,7 @@ vradi.mail.password=xxxxxxxx # Default session paragraph -vradi.session.defaultParagraph=Voici la liste des nouveaux appels d'offres.\n\nCordialement.\n\n\L'\u00e9quipe ???.\n +vradi.session.defaultParagraph=Voici la liste des nouveaux appels d'offres.\n\nCordialement.\n\n\L'\u00e9quipe Vradi.\n # Wikitty configuration wikitty.searchengine.solr.directory.factory=solr.StandardDirectoryFactory Modified: trunk/vradi-services/src/test/java/org/chorem/vradi/services/InternalUsageTest.java =================================================================== --- trunk/vradi-services/src/test/java/org/chorem/vradi/services/InternalUsageTest.java 2012-03-30 16:25:19 UTC (rev 169) +++ trunk/vradi-services/src/test/java/org/chorem/vradi/services/InternalUsageTest.java 2012-04-16 13:06:33 UTC (rev 170) @@ -1,18 +1,13 @@ package org.chorem.vradi.services; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.chorem.vradi.VradiTestAbstract; import org.chorem.vradi.beans.FormPagedResult; import org.chorem.vradi.beans.QueryParameters; -import org.chorem.vradi.entities.FieldTypeEnum; +import org.chorem.vradi.demo.StandaloneDataService; import org.chorem.vradi.entities.Form; -import org.chorem.vradi.entities.Infogene; -import org.chorem.vradi.entities.XmlFieldBinding; -import org.chorem.vradi.entities.XmlFieldBindingImpl; import org.chorem.vradi.entities.XmlStream; -import org.chorem.vradi.entities.XmlStreamImpl; import org.chorem.vradi.services.managers.BindingManager; import org.chorem.vradi.services.managers.FormManager; import org.chorem.vradi.services.managers.FormTypeManager; @@ -21,22 +16,15 @@ import org.chorem.vradi.services.tasks.ImportXmlStreamTasks; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -import org.nuiton.wikitty.entities.FieldType; -import org.nuiton.wikitty.entities.WikittyExtension; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; /** * @author Arnaud Thimel <thimel@codelutin.com> */ public class InternalUsageTest extends VradiTestAbstract { - public static final String APPEL_OFFRES_FORM_TYPE = "APPEL_OFFRES"; - protected BindingManager bindingManager; protected FormTypeManager formTypeManager; protected ThesaurusManager thesaurusManager; @@ -50,62 +38,13 @@ searchManager = new SearchManager(wikittyProxy, thesaurusManager); formManager = new FormManager(config, wikittyProxy, thesaurusManager, searchManager); bindingManager = new BindingManager(config, wikittyProxy, formTypeManager, formManager); - createAPPEL_OFFRESFormType(); - } - protected WikittyExtension createAPPEL_OFFRESFormType() throws VradiException { - Map<String, FieldType> fields = new LinkedHashMap<String, FieldType>(); - fields.put("URL_AO", FieldTypeEnum.STRING.getFieldType()); - fields.put("Contact_technique", FieldTypeEnum.STRING.getFieldType()); - fields.put("Site_acheteur", FieldTypeEnum.STRING.getFieldType()); - fields.put("Contact_administratif", FieldTypeEnum.STRING.getFieldType()); - fields.put("category", FieldTypeEnum.STRING.getFieldType()); - fields.put("documents", FieldTypeEnum.STRING.getFieldType()); - - return createFormType(APPEL_OFFRES_FORM_TYPE, fields); + StandaloneDataService standaloneDataService = new StandaloneDataService(config, bindingManager, formTypeManager, searchManager, thesaurusManager, wikittyProxy); + standaloneDataService.checkStreams(); } - - protected XmlFieldBinding createXmlFieldBinding(String xmlField, String formField) throws VradiException { - XmlFieldBinding binding = new XmlFieldBindingImpl(); - binding.setXmlField(ImmutableSet.of(xmlField)); - binding.setFormField(formField); - - XmlFieldBinding result = wikittyProxy.store(binding); - return result; - } - - public XmlStream createAPPEL_OFFRESXmlStream(String name, String url) throws Exception { - XmlStream stream = new XmlStreamImpl(); - stream.setName(name); - stream.setUrl(url); - stream.setFormTypeName(APPEL_OFFRES_FORM_TYPE); - // tous - stream.addXmlFieldBinding(createXmlFieldBinding("title", Infogene.FQ_FIELD_INFOGENE_OBJET).getWikittyId()); - stream.addXmlFieldBinding(createXmlFieldBinding("link", Infogene.FQ_FIELD_INFOGENE_SOURCEURL).getWikittyId()); - stream.addXmlFieldBinding(createXmlFieldBinding("description", Infogene.FQ_FIELD_INFOGENE_DESCRIPTION).getWikittyId()); - stream.addXmlFieldBinding(createXmlFieldBinding("pubDate", Form.FQ_FIELD_FORM_DATEPUB).getWikittyId()); - - // marchésonline - stream.addXmlFieldBinding(createXmlFieldBinding("category", APPEL_OFFRES_FORM_TYPE + ".category").getWikittyId()); - stream.addXmlFieldBinding(createXmlFieldBinding("dc:creator", Infogene.FQ_FIELD_INFOGENE_ENTITY).getWikittyId()); - // EdiTender - stream.addXmlFieldBinding(createXmlFieldBinding("guid", APPEL_OFFRES_FORM_TYPE + ".documents").getWikittyId()); - // EdiTender et e-marchespublics - stream.addXmlFieldBinding(createXmlFieldBinding("author", APPEL_OFFRES_FORM_TYPE + ".Contact_administratif").getWikittyId()); - // e-marchespublics - stream.addXmlFieldBinding(createXmlFieldBinding("department", Infogene.FQ_FIELD_INFOGENE_DEPARTMENT).getWikittyId()); - wikittyProxy.store(stream); - - return stream; - } - @Test - @Ignore public void registerMarchesOnlineXmlStream() throws Exception { - XmlStream marchesOnlineStream = createAPPEL_OFFRESXmlStream("Marchés Online", "http://www.marchesonline.com/mol/rss/appels-d-offres-services-informatiques...."); - XmlStream ediTenderStream = createAPPEL_OFFRESXmlStream("Edi Tender", "http://www.edi-tender.com/rss/marchespublics.xml"); - XmlStream eMarchesPublicsStream = createAPPEL_OFFRESXmlStream("e-Marchés Publics", "https://www.e-marchespublics.com/rss/rss.php?v=2&l=200&c=Service"); List<XmlStream> allXmlStreams = bindingManager.getAllXmlStreams(); Assert.assertEquals(3, allXmlStreams.size()); @@ -121,7 +60,8 @@ "J2EE", "JEE", "EJB", "swing", "struts", "hibernate", "Tomcat", "Jetty", "jboss", "jonas", - "maven" + "maven", + "Transbeauce" ); String query = Joiner.on(" OR ").join(keywords); FormPagedResult result = searchManager.findForms(new QueryParameters(query), new FormPagedResult());