Author: jcouteau Date: 2013-11-19 09:44:41 +0100 (Tue, 19 Nov 2013) New Revision: 690 Url: http://nuiton.org/projects/sandbox/repository/revisions/690 Log: Referentiel communes OSM Added: referentielCommunesOsm/ referentielCommunesOsm/pom.xml referentielCommunesOsm/src/ referentielCommunesOsm/src/main/ referentielCommunesOsm/src/main/java/ referentielCommunesOsm/src/main/java/org/ referentielCommunesOsm/src/main/java/org/nuiton/ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ArrondissementXMLReader.java referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/GPSPoint.java referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/OsmXMLReader.java referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ReferentielBuilder.java referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/Town.java referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownExportModel.java referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownInseeImportModel.java referentielCommunesOsm/src/main/resources/ referentielCommunesOsm/src/test/ referentielCommunesOsm/src/test/java/ Added: referentielCommunesOsm/pom.xml =================================================================== (Binary files differ) Property changes on: referentielCommunesOsm/pom.xml ___________________________________________________________________ Added: svn:mime-type + application/xml Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ArrondissementXMLReader.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ArrondissementXMLReader.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ArrondissementXMLReader.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,103 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ArrondissementXMLReader extends DefaultHandler { + //List to hold Employees object + protected List<Town> data = new ArrayList<Town>(); + protected Map<String, Town> centers = new HashMap<String, Town>(); + protected Town town = null; + protected String id = null; + + //getter method for employee list + public List<Town> getData() { + return data; + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) + throws SAXException { + + if (qName.equalsIgnoreCase("node")) { + //create a new Node and put it in map when leaving + id = attributes.getValue("id"); + String lat = attributes.getValue("lat"); + String lon = attributes.getValue("lon"); + //initialize Town object + town = new Town(); + town.setLatitude(Double.valueOf(lat)); + town.setLongitude(Double.valueOf(lon)); + + } else if (qName.equalsIgnoreCase("tag")) { + String key = attributes.getValue("k"); + if ("addr:city".equals(key)){ + String value = attributes.getValue("v"); + town.setArticle(value); + } else if ("addr:postcode".equals(key)){ + String value = attributes.getValue("v"); + town.setPostalCode(value); + } else if ("postal_code".equals(key)){ + String value = attributes.getValue("v"); + town.setPostalCode(value); + } else if ("name".equals(key)){ + String value = attributes.getValue("v"); + town.setName(value); + } else if ("ref:INSEE".equals(key)){ + String value = attributes.getValue("v"); + town.setInseeCode(value); + } + + } else if (qName.equalsIgnoreCase("relation")) { + town = new Town(); + + + } else if (qName.equalsIgnoreCase("member")) { + String role = attributes.getValue("role"); + if (role.equalsIgnoreCase("admin_centre")){ + String centerId = attributes.getValue("ref"); + Town adminCenter = centers.get(centerId); + + if (adminCenter != null){ + if (adminCenter.getPostalCode() != null) { + town.setPostalCode(adminCenter.getPostalCode()); + } + if (adminCenter.getLatitude() != 0.0){ + town.setLatitude(adminCenter.getLatitude()); + } + if (adminCenter.getLongitude() != 0.0){ + town.setLongitude(adminCenter.getLongitude()); + } + } + } + + } + } + + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + if (qName.equalsIgnoreCase("node")) { + //add Node object to maps + centers.put(id,town); + + } else if (qName.equalsIgnoreCase("relation")) { + town.computeTownName(); + town.computeDepartmentRegionCode(); + if (town.getName() != null && town.getPostalCode() != null && town.getPostalCode().length()==5) { + data.add(town); + } + } + } + + + @Override + public void characters(char ch[], int start, int length) throws SAXException { + } +} Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/GPSPoint.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/GPSPoint.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/GPSPoint.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,24 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + +public class GPSPoint { + + protected Double lat; + + protected Double lon; + + public Double getLat() { + return lat; + } + + public void setLat(Double lat) { + this.lat = lat; + } + + public Double getLon() { + return lon; + } + + public void setLon(Double lon) { + this.lon = lon; + } +} Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/OsmXMLReader.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/OsmXMLReader.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/OsmXMLReader.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,98 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import java.util.HashMap; +import java.util.Map; + +public class OsmXMLReader extends DefaultHandler{ + //List to hold Employees object + protected Map<String, Town> data = new HashMap<String, Town>(); + protected Map<String, Town> cityCenters = new HashMap<String, Town>(); + protected Town town = null; + protected String id = null; + + + //getter method for employee list + public Map<String, Town> getData() { + return data; + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) + throws SAXException { + + if (qName.equalsIgnoreCase("node")) { + //create a new Node and put it in map when leaving + id = attributes.getValue("id"); + String lat = attributes.getValue("lat"); + String lon = attributes.getValue("lon"); + //initialize Town object + town = new Town(); + town.setLatitude(Double.valueOf(lat)); + town.setLongitude(Double.valueOf(lon)); + + } else if (qName.equalsIgnoreCase("tag")) { + String key = attributes.getValue("k"); + if ("ref:INSEE".equals(key)){ + String value = attributes.getValue("v"); + town.setInseeCode(value); + } else if ("addr:postcode".equals(key)){ + String value = attributes.getValue("v"); + town.setPostalCode(value); + } else if ("postal_code".equals(key)){ + String value = attributes.getValue("v"); + town.setPostalCode(value); + } + + } else if (qName.equalsIgnoreCase("relation")) { + town = new Town(); + + + } else if (qName.equalsIgnoreCase("member")) { + String role = attributes.getValue("role"); + if (role.equalsIgnoreCase("admin_centre")){ + String id = attributes.getValue("ref"); + Town adminCenter = cityCenters.get(id); + + if (adminCenter != null){ + if (adminCenter.getPostalCode() != null) { + town.setPostalCode(adminCenter.getPostalCode()); + } + if (adminCenter.getLatitude() != 0.0){ + town.setLatitude(adminCenter.getLatitude()); + } + if (adminCenter.getLongitude() != 0.0){ + town.setLongitude(adminCenter.getLongitude()); + } + } + } + + } + } + + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + if (qName.equalsIgnoreCase("node")) { + //add Node object to maps + cityCenters.put(id,town); + + if (town.getInseeCode() != null) { + data.put(town.getInseeCode(), town); + } + + } else if (qName.equalsIgnoreCase("relation")) { + if (town.getInseeCode() != null) { + data.put(town.getInseeCode(), town); + } + } + } + + + @Override + public void characters(char ch[], int start, int length) throws SAXException { + } +} Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ReferentielBuilder.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ReferentielBuilder.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/ReferentielBuilder.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,91 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + +import org.nuiton.csv.Export; +import org.nuiton.csv.Import; + +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import java.io.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ReferentielBuilder { + + public static void main(String args[]) throws Exception{ + String inseeReferentiel = args[0]; + String osmReferentiel = args[1]; + String arrondissementReferentiel = args[2]; + String outComplete = args[3]; + String outUncomplete = args[4]; + + //list of complete towns + List<Town> completeTowns = new ArrayList<Town>(); + + //list of uncomplete towns + List<Town> uncompleteTowns = new ArrayList<Town>(); + + //prepare XML import + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + SAXParser saxParser = saxParserFactory.newSAXParser(); + + //import osmReferentiel + OsmXMLReader handler = new OsmXMLReader(); + saxParser.parse(new File(osmReferentiel), handler); + Map<String, Town> osmData = handler.getData(); + System.out.println(osmData.size() + " towns coming from OSM"); + + //import arrondissements referentiel + ArrondissementXMLReader arrondissementHandler = new ArrondissementXMLReader(); + saxParser.parse(new File(arrondissementReferentiel), arrondissementHandler); + List<Town> arrondissements = arrondissementHandler.getData(); + + //Prepare insee input stream + File inseeReferentialFile = new File (inseeReferentiel); + InputStream inseeReferentialStream = new FileInputStream(inseeReferentialFile); + + //Import Insee referentiel + Import<Town> importer = Import.newImport(new TownInseeImportModel(), new InputStreamReader(inseeReferentialStream)); + + //iterate on towns to put lat,lon and postalCode coming form OSM and store complete towns on completeTowns and + // uncomplete towns in uncompleteTowns + for (Town town:importer){ + town.computeTownName(); + town.computeInseeCode(); + + if (osmData.containsKey(town.getInseeCode())){ + Town osmTown = osmData.get(town.getInseeCode()); + + if (osmTown.getPostalCode() != null) { + town.setPostalCode(osmTown.getPostalCode()); + town.setLongitude(osmTown.getLongitude()); + town.setLatitude(osmTown.getLatitude()); + completeTowns.add(town); + } else { + uncompleteTowns.add(town); + //System.out.println("No postal code for : " + town.getName() + " - "+town.getDepartment()); + } + } else { + uncompleteTowns.add(town); + //System.out.println("No INSEE code for : " + town.getName()); + } + } + + System.out.println(completeTowns.size() + " complete towns"); + System.out.println(uncompleteTowns.size() + " uncomplete towns"); + System.out.println(arrondissements.size() + " arrondissements"); + + completeTowns.addAll(arrondissements); + + // export completeTowns + File completeTownOutputFile = new File(outComplete); + Export.exportToFile(new TownExportModel(), completeTowns, completeTownOutputFile, Charset.forName("UTF-8")); + + // export uncompleteTowns + File uncompleteTownOutputFile = new File(outUncomplete); + Export.exportToFile(new TownExportModel(), uncompleteTowns, uncompleteTownOutputFile, Charset.forName("UTF-8")); + + Export.exportToFile(new TownExportModel(), arrondissements, new File("arr.csv"), Charset.forName("UTF-8")); + } +} Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/Town.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/Town.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/Town.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,132 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + +import com.google.common.base.Strings; +import org.apache.commons.lang3.StringUtils; + +public class Town { + protected String inseeCode; + protected String regionCode; + protected String department; + protected String postalCode; + protected String townCode; + protected String name; + protected String article; + protected double latitude; + protected double longitude; + + public String getInseeCode() { + return inseeCode; + } + + public void setInseeCode(String inseeCode) { + this.inseeCode = inseeCode; + } + + public String getRegionCode() { + return regionCode; + } + + public void setRegionCode(String regionCode) { + this.regionCode = regionCode; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getPostalCode() { + return postalCode; + } + + public void setPostalCode(String postalCode) { + this.postalCode = postalCode; + } + + public double getLatitude() { + return latitude; + } + + public void setLatitude(double latitude) { + this.latitude = latitude; + } + + public double getLongitude() { + return longitude; + } + + public void setLongitude(double longitude) { + this.longitude = longitude; + } + + public String getTownCode() { + return townCode; + } + + public void setTownCode(String townCode) { + this.townCode = townCode; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getArticle() { + return article; + } + + public void setArticle(String article) { + this.article = article; + } + + protected void computeInseeCode(){ + if (inseeCode == null) { + if (department.length() == 3) { // Cas particulier des DOM/TOM + townCode = Strings.padStart(townCode, 2, '0'); + } else { + townCode = Strings.padStart(townCode, 3, '0'); + } + inseeCode = department + townCode; + } + inseeCode = Strings.padStart(inseeCode, 5, '0'); + + } + + protected void computeTownName(){ + if (StringUtils.isNotBlank(article)) { + article = article.trim(); + if (article.startsWith("(")) { + article = article.substring(1); + } + if (article.endsWith(")")) { + article = article.substring(0, article.length() - 1); + } + if (!article.endsWith("'")) { + article += " "; + } + name = article + name; + } + } + + protected void computeDepartmentRegionCode(){ + if (postalCode!= null){ + if (postalCode.startsWith("75")){ + department = "75"; + regionCode = "11"; + } else if (postalCode.startsWith("69")){ + department = "69"; + regionCode = "82"; + } else if (postalCode.startsWith("13")){ + department = "13"; + regionCode = "93"; + } + } + } +} Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownExportModel.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownExportModel.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownExportModel.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,39 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + +import org.nuiton.csv.Common; +import org.nuiton.csv.ExportModel; +import org.nuiton.csv.ExportableColumn; +import org.nuiton.csv.ModelBuilder; + +public class TownExportModel implements ExportModel<Town> { + + @Override + public char getSeparator() { + return ';'; + } + + @Override + public Iterable<ExportableColumn<Town, Object>> getColumnsForExport() { + + ModelBuilder<Town> modelBuilder = new ModelBuilder<Town>(); + + modelBuilder.newColumnForExport("INSEE_CODE", "inseeCode"); + + modelBuilder.newColumnForExport("REGION_CODE", "regionCode"); + + modelBuilder.newColumnForExport("DEPARTMENT", "department"); + + modelBuilder.newColumnForExport("POSTAL_CODE", "postalCode"); + + modelBuilder.newColumnForExport("NAME", "name"); + + modelBuilder.newColumnForExport("LAT", "latitude", Common.DOUBLE); + + modelBuilder.newColumnForExport("LON", "longitude", Common.DOUBLE); + + + return (Iterable) modelBuilder.getColumnsForExport(); + + } + +} \ No newline at end of file Added: referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownInseeImportModel.java =================================================================== --- referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownInseeImportModel.java (rev 0) +++ referentielCommunesOsm/src/main/java/org/nuiton/sandbox/referentielCommunesOsm/TownInseeImportModel.java 2013-11-19 08:44:41 UTC (rev 690) @@ -0,0 +1,52 @@ +package org.nuiton.sandbox.referentielCommunesOsm; + + +import org.nuiton.csv.Common; +import org.nuiton.csv.ImportModel; +import org.nuiton.csv.ImportableColumn; +import org.nuiton.csv.ModelBuilder; + +import java.util.List; + +public class TownInseeImportModel implements ImportModel<Town> { + + @Override + public char getSeparator() { + return '\t'; + } + + @Override + public void pushCsvHeaderNames(List<String> headerNames) { + //Do nothing + } + + @Override + public Town newEmptyInstance() { + return new Town(); + } + + + @Override + public Iterable<ImportableColumn<Town, Object>> getColumnsForImport() { + + ModelBuilder<Town> modelBuilder = new ModelBuilder<Town>(); + + modelBuilder.newIgnoredColumn("CDC"); + modelBuilder.newIgnoredColumn("CHEFLIEU"); + modelBuilder.newMandatoryColumn("REG", "regionCode", Common.STRING); + modelBuilder.newMandatoryColumn("DEP", "department", Common.STRING); + modelBuilder.newMandatoryColumn("COM", "townCode", Common.STRING); + modelBuilder.newIgnoredColumn("AR"); + modelBuilder.newIgnoredColumn("CT"); + modelBuilder.newIgnoredColumn("TNCC"); + modelBuilder.newIgnoredColumn("ARTMAJ"); + modelBuilder.newIgnoredColumn("NCC"); + modelBuilder.newMandatoryColumn("ARTMIN", "article", Common.STRING); + modelBuilder.newMandatoryColumn("NCCENR", "name", Common.STRING); + + + return (Iterable) modelBuilder.getColumnsForImport(); + + } + +}