From b2d1d6df1eaebc097d8eec5c8bf80f0430b800c9 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 20 Nov 2015 12:24:26 +0000 Subject: [PATCH 01/22] Script for importing UK postcodes from ONSPD Importing everything from one source reduces what we have to download and we have to download ONSPD to get NI and Crown dependencies which aren't in Code Point Open. Other advantages are that the ONSPD has all live and terminated postcodes in it and (as of Aug 2015 release at least) everything is a gss code, rather than a mix of ons + gss. We add a toggle to allow importing terminated postcodes as it can be useful to have old postcodes in the db to allow searching on old addresses. Note that although code point open doesn't include terminated postcodes we can end up with them in our dataset, but only if we had a long-lived database that imported multiple releases of the dataset. For example, we import the May 2012 dataset and then when the next one comes out in Aug 2012 we import that - our db will have in it the postcodes that were terminated between those two releases, but it won't have any that were terminated before May 2012. This leads to the situation where a db rebuilt from scratch using the current dataset would have a different set of postcodes to one that had been around for a few years having had releases imported as they arrive. Notionally both represent the current data, but one has more postcodes. Using the ONSPD and allowing terminated postcodes fixes this problem. --- ...mapit_UK_import_nspd_crown_dependencies.py | 13 +- .../commands/mapit_UK_import_onspd.py | 112 ++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 mapit_gb/management/commands/mapit_UK_import_onspd.py diff --git a/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py b/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py index 0ba8b7db..63714d8d 100644 --- a/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py +++ b/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py @@ -3,15 +3,24 @@ # http://www.ons.gov.uk/about-statistics/geography/products/geog-products-postcode/nspd/ from mapit.management.commands.mapit_import_postal_codes import Command - +from optparse import make_option class Command(Command): help = 'Imports Crown Dependency postcodes from the NSPD' args = '' option_defaults = {'strip': True, 'location': False} + option_list = Command.option_list + ( + make_option( + '--allow-terminated-postcodes', + action='store_true', + dest='include-terminated', + default=False, + help='Set if you want to import terminated postcodes' + ), + ) def pre_row(self, row, options): - if row[4]: + if row[4] and not options['include-terminated']: return False # Terminated postcode if self.code[0:2] not in ('GY', 'JE', 'IM'): return False # Only importing Crown dependencies from NSPD diff --git a/mapit_gb/management/commands/mapit_UK_import_onspd.py b/mapit_gb/management/commands/mapit_UK_import_onspd.py new file mode 100644 index 00000000..9b474660 --- /dev/null +++ b/mapit_gb/management/commands/mapit_UK_import_onspd.py @@ -0,0 +1,112 @@ +# This script is used to import Northern Ireland postcode information from the +# National Statistics Postcode Database. +# http://www.ons.gov.uk/about-statistics/geography/products/geog-products-postcode/nspd/ +# +# The fields of ONSPD Open CSV file are (as of Aug 2015 release) +# 1. Unit postcode - 7 character version +# 2. Unit postcode - 8 character version +# 3. Unit postcode - variable length (e-Gif) version +# 4. Date of introduction +# 5. Date of termination +# 6. County +# 7. Local authority district (LAD)/unitary authority (UA)/ metropolitan +# district (MD)/ London borough (LB)/ council area (CA)/district council +# area (DCA) +# 8. (Electoral) ward/division +# 9. Postcode user type +# 10. National grid reference - Easting +# 11. National grid reference - Northing +# 12. Grid reference positional quality indicator +# 13. Former Strategic health authority (SHA)/ +# local health board (LHB)/ +# health board (HB)/ +# health authority (HA)/ +# health & social care board (HSCB) +# 14. Pan SHA +# 15. Country +# 16. Region (formerly GOR) +# 17. Standard (statistical) region (SSR) +# 18. Westminster parliamentary constituency +# 19. European Electoral Region (EER) +# 20. Local Learning and Skills Council (LLSC)/ +# Dept. of Children, Education, Lifelong Learning and Skills (DCELLS)/ +# Enterprise Region (ER) +# 21. Travel-to-work area (TTWA) +# 22. Primary Care Trust (PCT)/ +# Care Trust/ +# Care Trust Plus (CT)/ +# local health board (LHB)/ +# community health partnership (CHP)/ +# local commissioning group (LCG)/ +# primary healthcare directorate (PHD) +# 23. LAU2 areas +# 24. 1991 Census Enumeration District (ED) +# 25. 1991 Census Enumeration District (ED) +# 26. ED positional quality indicator +# 27. Previous Strategic health authority (SHA)/ +# health board (HB)/ +# health authority (HA)/ +# health and social services board (HSSB) +# 28. Local Education Authority (LEA)/ +# Education and Library Board (ELB) +# 29. Health Authority 'old-style' +# 30. 1991 ward (Census code range) +# 31. 1991 ward (OGSS code range) +# 32. 1998 ward +# 33. 2005 'statistical' ward (England and Wales only) +# 34. 2001 Census output area +# 35. Census Area Statistics (CAS) ward +# 36. National park +# 37. 2001 Census lower layer super output area (LSOA) +# 38. 2001 Census middle layer super output area (MSOA) +# 39. 2001 Census urban/rural indicator +# 40. 2001 Census output area classification (OAC) +# 41. 'Old' Primary Care Trust (PCT)/ +# Local Health Board (LHB)/ +# Care Trust (CT) +# 42. 2011 Census output area (OA)/ +# small area +# 43. 2011 Census lower layer super output area (LSOA)/ +# data zone (DZ)/ SOA +# 44. 2011 Census middle layer super output area (MSOA)/ +# intermediate zone (IZ) +# 45. Parish (England)/ community (Wales) +# 46. 2011 Census workplace zone +# 47. Clinical Commissioning Group (CCG)/ +# local health board (LHB)/ +# community health partnership (CHP)/ +# local commissioning group (LCG)/ +# primary healthcare directorate (PHD) +# 48. Built-up area +# 49. Built-up area sub-division +# 50. 2011 Census rural-urban classification +# 51. 2011 Census output area classification (OAC) +# 52. Decimal degrees latitude +# 53. Decimal degrees longitude + +from optparse import make_option +from mapit.management.commands.mapit_import_postal_codes import Command + + +class Command(Command): + help = 'Imports UK postcodes from the NSPD, excluding NI and Crown Dependencies' + args = '' + option_defaults = {'header-row': True, 'strip': True, 'srid': 27700, 'coord-field-lon': 10, 'coord-field-lat': 11} + option_list = Command.option_list + ( + make_option( + '--allow-terminated-postcodes', + action='store_true', + dest='include-terminated', + default=False, + help='Set if you want to import terminated postcodes' + ), + ) + + def pre_row(self, row, options): + if row[4] and not options['include-terminated']: + return False # Terminated postcode + if row[11] == '9': + return False # PO Box etc. + if self.code[0:2] in ('GY', 'JE', 'IM', 'BT'): + return False # NI and channel islands handled by other commands + return True From 74aa67316947ad67b822a8f42437284d25707b72 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Tue, 24 Nov 2015 16:53:37 +0000 Subject: [PATCH 02/22] Allow detecting location availability per postcode row If `--no-location` is not set we would try to detect a location for each row, and this would break if the location fields could not be coerced into floats. Some datasets mix location and non-location postal codes and to import them all we have to filter the data and run the importer twice. This change allows individual importers to implement `location_available_for_row` to say if the supplied row has location data or not. The method is called on each row and will run the `--no-location` path if we can't extract location fields for that row. If `--no-location` is set, we always run that path, regardless of the `location_available_for_row` value. --- mapit/management/commands/mapit_import_postal_codes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mapit/management/commands/mapit_import_postal_codes.py b/mapit/management/commands/mapit_import_postal_codes.py index 445c9349..73a19f8c 100644 --- a/mapit/management/commands/mapit_import_postal_codes.py +++ b/mapit/management/commands/mapit_import_postal_codes.py @@ -109,8 +109,11 @@ def pre_row(self, row, options): def post_row(self, pc): return True + def location_available_for_row(self, row): + return True + def handle_row(self, row, options): - if not options['location']: + if not options['location'] or not self.location_available_for_row(row): return self.do_postcode() if not options['coord-field-lon']: From 32d772e62bae8cec3dff3b3995010c9880143682 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 20 Nov 2015 12:39:29 +0000 Subject: [PATCH 03/22] Provide fixture data for NI council / electoral / wards In April 2015 councils and wards changed in Northern Ireland so the old ni-electoral-areas data files no longer represent the truth. The new ni-electoral-areas-2015 file provides the names and GSS codes of the new Districts, Electoral Areas, and Wards of Northern Ireland following the Apr 2015 reorganisation. We synthesized this from a few datasets. ** The District -> Electoral Area -> Ward breakdown is taken directly from the legislation[2] - although this only contains the names ** The GSS codes of the Districts and Wards are taken from the "Wards (2015) to district council areas (2015) NI lookup"[3] dataset provided by the ONS. ** The GSS codes of the Electoral areas are taken from running sparql queries against the ONS Linked Data Portal[4]. The query used was: PREFIX rdf: PREFIX geography: PREFIX statistical-entity: SELECT DISTINCT ?item WHERE { ?item rdf:type geography:statistical-geography . ?item statistical-entity:code . } ORDER BY ASC(?code) LIMIT 100 OFFSET 0 N10 is the code for a NI Electoral Area (found by looking up one of the areas by name and investigating). We then extract the data for each entry in this result set and extract the GSS code to match up with the name. This data will be used to help import NI areas from the OSNI boundary datasets which do not all contain GSS codes. [1]: https://geoportal.statistics.gov.uk/geoportal/catalog/search/resource/details.page?uuid=%7B2196C1D5-6A11-47DE-BD0E-26311E3D6D9F%7D [2]: http://www.legislation.gov.uk/uksi/2014/270/made [3]: https://geoportal.statistics.gov.uk/geoportal/catalog/search/resource/details.page?uuid=%7BFE83C43C-9403-408C-833C-367BC56659C9%7D [4]: http://statistics.data.gov.uk/ --- mapit_gb/data/ni-electoral-areas-2015.csv | 464 ++++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 mapit_gb/data/ni-electoral-areas-2015.csv diff --git a/mapit_gb/data/ni-electoral-areas-2015.csv b/mapit_gb/data/ni-electoral-areas-2015.csv new file mode 100644 index 00000000..697f3c01 --- /dev/null +++ b/mapit_gb/data/ni-electoral-areas-2015.csv @@ -0,0 +1,464 @@ +# from http://www.legislation.gov.uk/uksi/2014/270/schedule/made,,,,, +District,District GSS Code,Electoral area,Electoral area GSS code,Ward,Ward GSS code +Antrim and Newtownabbey,N09000001,Dunsilly,N10000104,Cranfield,N08000115 +,,,,Parkgate,N08000129 +,,,,Randalstown,N08000130 +,,,,Shilvodan,N08000133 +,,,,Toome,N08000138 +,,Antrim,N10000102,Antrim Centre,N08000103 +,,,,Fountain Hill,N08000119 +,,,,Greystone,N08000122 +,,,,Springfarm,N08000134 +,,,,Steeple,N08000135 +,,,,Stiles,N08000136 +,,Airport,N10000101,Aldergrove,N08000102 +,,,,Clady,N08000113 +,,,,Crumlin,N08000116 +,,,,Mallusk,N08000125 +,,,,Templepatrick,N08000137 +,,Ballyclare,N10000103,Ballyclare East,N08000104 +,,,,Ballyclare West,N08000105 +,,,,Ballynure,N08000108 +,,,,Ballyrobert,N08000109 +,,,,Doagh,N08000117 +,,Three Mile Water,N10000107,Ballyduff,N08000106 +,,,,Fairview,N08000118 +,,,,Jordanstown,N08000124 +,,,,Monkstown,N08000126 +,,,,Mossley,N08000127 +,,,,Rostulla,N08000132 +,,Macedon,N10000106,Abbey,N08000101 +,,,,Carnmoney Hill,N08000112 +,,,,O’Neill,N08000128 +,,,,Rathcoole,N08000131 +,,,,Valley,N08000139 +,,,,Whitehouse,N08000140 +,,Glengormley Urban,N10000105,Ballyhenry,N08000107 +,,,,Burnthill,N08000110 +,,,,Carnmoney,N08000111 +,,,,Collinbridge,N08000114 +,,,,Glebe,N08000120 +,,,,Glengormley,N08000121 +,,,,Hightown,N08000123 +"Armagh, Banbridge and Craigavon",N09000002,Armagh,N10000201,Blackwatertown,N08000207 +,,,,Cathedral,N08000210 +,,,,Demesne,N08000213 +,,,,Keady,N08000220 +,,,,Navan,N08000231 +,,,,The Mall,N08000240 +,,Cusher,N10000204,Hamiltonsbawn,N08000219 +,,,,Markethill,N08000229 +,,,,Richhill,N08000235 +,,,,Seagahan,N08000236 +,,,,Tandragee,N08000238 +,,Portadown,N10000207,Ballybay,N08000202 +,,,,Corcrain,N08000211 +,,,,Killycomain,N08000222 +,,,,Loughgall,N08000226 +,,,,Mahon,N08000228 +,,,,The Birches,N08000239 +,,Craigavon,N10000203,Bleary,N08000208 +,,,,Brownlow,N08000209 +,,,,Craigavon Centre,N08000212 +,,,,Derrytrasna,N08000214 +,,,,Kernan,N08000221 +,,Lurgan,N10000206,Aghagallon,N08000201 +,,,,Knocknashane,N08000223 +,,,,Lough Road,N08000224 +,,,,Magheralin,N08000227 +,,,,Mourneview,N08000230 +,,,,Parklake,N08000232 +,,,,Shankill,N08000237 +,,Lagan River,N10000205,Donaghcloney,N08000215 +,,,,Dromore,N08000216 +,,,,Gransha,N08000218 +,,,,Quilly,N08000233 +,,,,Waringstown,N08000241 +,,Banbridge,N10000202,Banbridge East,N08000203 +,,,,Banbridge North,N08000204 +,,,,Banbridge South,N08000205 +,,,,Banbridge West,N08000206 +,,,,Gilford,N08000217 +,,,,Loughbrickland,N08000225 +,,,,Rathfriland,N08000234 +Belfast,N09000003,Castle,N10000304,Bellevue,N08000309 +,,,,Cavehill,N08000314 +,,,,Chichester Park,N08000316 +,,,,Duncairn,N08000322 +,,,,Fortwilliam,N08000328 +,,,,Innisfayle,N08000332 +,,Oldpark,N10000308,Ardoyne,N08000302 +,,,,Ballysillan,N08000306 +,,,,Cliftonville,N08000317 +,,,,Legoniel,N08000336 +,,,,New Lodge,N08000340 +,,,,Water Works,N08000357 +,,Court,N10000306,Ballygomartin,N08000303 +,,,,Clonard,N08000318 +,,,,Falls,N08000324 +,,,,Forth River,N08000327 +,,,,Shankill,N08000348 +,,,,Woodvale,N08000360 +,,Black Mountain,N10000302,Andersonstown,N08000301 +,,,,Ballymurphy,N08000305 +,,,,Beechmount,N08000307 +,,,,Collin Glen,N08000319 +,,,,Falls Park,N08000325 +,,,,Shaw’s Road,N08000349 +,,,,Turf Lodge,N08000354 +,,Collin,N10000305,Dunmurry,N08000323 +,,,,Ladybrook,N08000334 +,,,,Lagmore,N08000335 +,,,,Poleglass,N08000343 +,,,,Stewartstown,N08000350 +,,,,Twinbrook,N08000355 +,,Balmoral,N10000301,Belvoir,N08000311 +,,,,Finaghy,N08000326 +,,,,Malone,N08000337 +,,,,Musgrave,N08000339 +,,,,Upper Malone,N08000356 +,,Botanic,N10000303,Blackstaff,N08000312 +,,,,Central,N08000315 +,,,,Ormeau,N08000342 +,,,,Stranmillis,N08000352 +,,,,Windsor,N08000358 +,,Lisnasharragh,N10000307,Cregagh,N08000321 +,,,,Hillfoot,N08000331 +,,,,Merok,N08000338 +,,,,Orangefield,N08000341 +,,,,Ravenhill,N08000344 +,,,,Rosetta,N08000345 +,,Ormiston,N10000309,Belmont,N08000310 +,,,,Garnerville,N08000329 +,,,,Gilnahirk,N08000330 +,,,,Knock,N08000333 +,,,,Sandown,N08000346 +,,,,Shandon,N08000347 +,,,,Stormont,N08000351 +,,Titanic,N10000310,Ballymacarrett,N08000304 +,,,,Beersbridge,N08000308 +,,,,Bloomfield,N08000313 +,,,,Connswater,N08000320 +,,,,Sydenham,N08000353 +,,,,Woodstock,N08000359 +Causeway Coast and Glens,N09000004,The Glens,N10000407,Ballycastle,N08000404 +,,,,Kinbane,N08000425 +,,,,Loughguile and Stranocum,N08000426 +,,,,Lurigethan,N08000427 +,,,,Torr Head and Rathlin,N08000437 +,,Causeway,N10000404,Atlantic,N08000403 +,,,,Dervock,N08000413 +,,,,Dundooan,N08000415 +,,,,Giant’s Causeway,N08000420 +,,,,Hopefield,N08000423 +,,,,Portrush and Dunluce,N08000431 +,,,,Portstewart,N08000432 +,,Ballymoney,N10000401,Ballymoney East,N08000406 +,,,,Ballymoney North,N08000407 +,,,,Ballymoney South,N08000408 +,,,,Clogh Mills,N08000411 +,,,,Dunloy,N08000417 +,,,,Rasharkin,N08000434 +,,,,Route,N08000436 +,,Coleraine,N10000405,Churchland,N08000410 +,,,,Mountsandel,N08000430 +,,,,Quarry,N08000433 +,,,,University,N08000438 +,,,,Waterside,N08000439 +,,,,Windy Hall,N08000440 +,,Bann,N10000402,Aghadowey,N08000401 +,,,,Castlerock,N08000409 +,,,,Garvagh,N08000419 +,,,,Kilrea,N08000424 +,,,,Macosquin,N08000428 +,,Benbradagh,N10000403,Altahullion,N08000402 +,,,,Ballykelly,N08000405 +,,,,Dungiven,N08000416 +,,,,Feeny,N08000418 +,,,,Greysteel,N08000421 +,,Limavady,N10000406,Coolessan,N08000412 +,,,,Drumsurn,N08000414 +,,,,Greystone,N08000422 +,,,,Magilligan,N08000429 +,,,,Roeside,N08000435 +Derry and Strabane,N09000005,Ballyarnett,N10000501,Carn Hill,N08000505 +,,,,Culmore,N08000513 +,,,,Galliagh,N08000521 +,,,,Shantallow,N08000531 +,,,,Shantallow East,N08000532 +,,,,Skeoge,N08000535 +,,Foyleside,N10000504,Ballymagroarty,N08000503 +,,,,Foyle Springs,N08000520 +,,,,Madam’s Bank,N08000526 +,,,,Northland,N08000529 +,,,,Springtown,N08000537 +,,The Moor,N10000506,Brandywell,N08000504 +,,,,City Walls,N08000508 +,,,,Creggan,N08000511 +,,,,Creggan South,N08000512 +,,,,Sheriff’s Mountain,N08000533 +,,Waterside,N10000507,Caw,N08000507 +,,,,Clondermot,N08000510 +,,,,Drumahoe,N08000514 +,,,,Ebrington,N08000516 +,,,,Kilfennan,N08000524 +,,,,Lisnagelvin,N08000525 +,,,,Victoria,N08000540 +,,Faughan,N10000503,Claudy,N08000509 +,,,,Eglinton,N08000517 +,,,,Enagh,N08000518 +,,,,New Buildings,N08000527 +,,,,Slievekirk,N08000536 +,,Sperrin,N10000505,Artigarvan,N08000501 +,,,,Ballycolman,N08000502 +,,,,Dunnamanagh,N08000515 +,,,,Glenelly Valley,N08000523 +,,,,Park,N08000530 +,,,,Strabane North,N08000538 +,,,,Strabane West,N08000539 +,,Derg,N10000502,Castlederg,N08000506 +,,,,Finn,N08000519 +,,,,Glenderg,N08000522 +,,,,Newtownstewart,N08000528 +,,,,Sion Mills,N08000534 +Fermanagh and Omagh,N09000006,Erne West,N10000604,Belcoo and Garrison,N08000602 +,,,,"Boho, Cleenish and Letterbreen",N08000605 +,,,,Derrygonnelly,N08000611 +,,,,Derrylin,N08000612 +,,,,Florence Court and Kinawley,N08000621 +,,Erne North,N10000603,Ballinamallard,N08000601 +,,,,Belleek and Boa,N08000603 +,,,,Ederney and Kesh,N08000617 +,,,,Irvinestown,N08000624 +,,,,Tempo,N08000638 +,,Enniskillen,N10000601,Castlecoole,N08000608 +,,,,Erne,N08000618 +,,,,Lisbellaw,N08000626 +,,,,Lisnarrick,N08000627 +,,,,Portora,N08000633 +,,,,Rossorry,N08000635 +,,West Tyrone,N10000607,Dromore,N08000614 +,,,,Drumquin,N08000616 +,,,,Fairy Water,N08000619 +,,,,Fintona,N08000620 +,,,,Newtownsaville,N08000631 +,,,,Trillick,N08000640 +,,Omagh,N10000606,Camowen,N08000607 +,,,,Coolnagard,N08000609 +,,,,Dergmoney,N08000610 +,,,,Gortrush,N08000623 +,,,,Killyclogher,N08000625 +,,,,Strule,N08000637 +,,Mid Tyrone,N10000605,Beragh,N08000604 +,,,,Drumnakilly,N08000615 +,,,,Gortin,N08000622 +,,,,Owenkillew,N08000632 +,,,,Sixmilecross,N08000636 +,,,,Termon,N08000639 +,,Erne East,N10000602,Brookeborough,N08000606 +,,,,Donagh,N08000613 +,,,,Lisnaskea,N08000628 +,,,,Maguiresbridge,N08000629 +,,,,Newtownbutler,N08000630 +,,,,Rosslea,N08000634 +Lisburn and Castlereagh,N09000007,Killultagh,N10000705,Ballinderry,N08000701 +,,,,Glenavy,N08000718 +,,,,Maghaberry,N08000730 +,,,,Stonyford,N08000738 +,,,,White Mountain,N08000740 +,,Lisburn South,N10000707,Ballymacash,N08000703 +,,,,Ballymacoss,N08000705 +,,,,Knockmore,N08000725 +,,,,Lagan Valley,N08000727 +,,,,Lisnagarvey,N08000729 +,,,,Old Warren,N08000736 +,,Downshire East,N10000703,Ballymacbrennan,N08000704 +,,,,Dromara,N08000713 +,,,,Drumbo,N08000714 +,,,,Hillhall,N08000722 +,,,,Ravernet,N08000737 +,,Downshire West,N10000704,Blaris,N08000707 +,,,,Hillsborough,N08000723 +,,,,Lagan,N08000726 +,,,,Maze,N08000732 +,,,,Moira,N08000733 +,,Lisburn North,N10000706,Derryaghy,N08000712 +,,,,Harmony Hill,N08000720 +,,,,Hilden,N08000721 +,,,,Lambeg,N08000728 +,,,,Magheralave,N08000731 +,,,,Wallace Park,N08000739 +,,Castlereagh South,N10000702,Beechill,N08000706 +,,,,Cairnshill,N08000708 +,,,,Carryduff East,N08000710 +,,,,Carryduff West,N08000711 +,,,,Galwally,N08000717 +,,,,Knockbracken,N08000724 +,,,,Newtownbreda,N08000735 +,,Castlereagh East,N10000701,Ballyhanwood,N08000702 +,,,,Carrowreagh,N08000709 +,,,,Dundonald,N08000715 +,,,,Enler,N08000716 +,,,,Graham’s Bridge,N08000719 +,,,,Moneyreagh,N08000734 +Mid and East Antrim,N09000008,Knockagh,N10000806,Burleigh Hill,N08000810 +,,,,Gortalee,N08000823 +,,,,Greenisland,N08000825 +,,,,Sunnylands,N08000836 +,,,,Woodburn,N08000840 +,,Carrick Castle,N10000804,Boneybefore,N08000807 +,,,,Castle,N08000813 +,,,,Kilroot,N08000828 +,,,,Love Lane,N08000831 +,,,,Victoria,N08000838 +,,Larne Lough,N10000807,Ballycarry and Glynn,N08000805 +,,,,Curran and Inver,N08000817 +,,,,Islandmagee,N08000826 +,,,,Kilwaughter,N08000829 +,,,,Whitehead South,N08000839 +,,Coast Road,N10000805,Cairncastle,N08000811 +,,,,Carnlough and Glenarm,N08000812 +,,,,Craigyhill,N08000815 +,,,,Gardenmore,N08000820 +,,,,The Maidens,N08000837 +,,Braid,N10000803,Ballee and Harryville,N08000804 +,,,,Broughshane,N08000809 +,,,,Glenravel,N08000821 +,,,,Glenwhirry,N08000822 +,,,,Kells,N08000827 +,,,,Kirkinriola,N08000830 +,,,,Slemish,N08000835 +,,Ballymena,N10000801,Academy,N08000801 +,,,,Ardeevin,N08000803 +,,,,Ballykeel,N08000806 +,,,,Braidwater,N08000808 +,,,,Castle Demesne,N08000814 +,,,,Fair Green,N08000818 +,,,,Park,N08000833 +,,Bannside,N10000802,Ahoghill,N08000802 +,,,,Cullybackey,N08000816 +,,,,Galgorm,N08000819 +,,,,Grange,N08000824 +,,,,Maine,N08000832 +,,,,Portglenone,N08000834 +Mid Ulster,N09000009,Carntogher,N10000901,Lower Glenshane,N08000926 +,,,,Maghera,N08000927 +,,,,Swatragh,N08000934 +,,,,Tamlaght O’Crilly,N08000935 +,,,,Valley,N08000939 +,,Moyola,N10000906,Ballymaguigan,N08000905 +,,,,Bellaghy,N08000907 +,,,,Castledawson,N08000910 +,,,,Draperstown,N08000919 +,,,,Tobermore,N08000937 +,,Magherafelt,N10000905,Coolshinny,N08000917 +,,,,Glebe,N08000921 +,,,,Lissan,N08000924 +,,,,The Loup,N08000936 +,,,,Town Parks East,N08000938 +,,Cookstown,N10000903,Coagh,N08000911 +,,,,Cookstown East,N08000914 +,,,,Cookstown South,N08000915 +,,,,Cookstown West,N08000916 +,,,,Loughry,N08000925 +,,,,Oaklands,N08000931 +,,,,Pomeroy,N08000932 +,,Torrent,N10000907,Ardboe,N08000901 +,,,,Coalisland North,N08000912 +,,,,Coalisland South,N08000913 +,,,,Donaghmore,N08000918 +,,,,Stewartstown,N08000933 +,,,,Washing Bay,N08000940 +,,Dungannon,N10000904,Ballysaggart,N08000906 +,,,,Killyman,N08000922 +,,,,Killymeal,N08000923 +,,,,Moy,N08000928 +,,,,Moygashel,N08000929 +,,,,Mullaghmore,N08000930 +,,Clogher Valley,N10000902,Augher and Clogher,N08000902 +,,,,Aughnacloy,N08000903 +,,,,Ballygawley,N08000904 +,,,,Caledon,N08000908 +,,,,Castlecaulfield,N08000909 +,,,,Fivemiletown,N08000920 +"Newry, Mourne and Down",N09000010,Slieve Gullion,N10001006,Bessbrook,N08001007 +,,,,Camlough,N08001010 +,,,,Crossmaglen,N08001014 +,,,,Forkhill,N08001023 +,,,,Mullaghbane,N08001031 +,,,,Newtownhamilton,N08001033 +,,,,Whitecross,N08001041 +,,Newry,N10001003,Abbey,N08001001 +,,,,Ballybot,N08001003 +,,,,Damolly,N08001015 +,,,,Drumalane,N08001019 +,,,,Fathom,N08001022 +,,,,St. Patrick’s,N08001037 +,,Crotlieve,N10001001,Burren,N08001009 +,,,,Derryleckagh,N08001017 +,,,,Hilltown,N08001024 +,,,,Mayobridge,N08001030 +,,,,Rostrevor,N08001035 +,,,,Warrenpoint,N08001040 +,,The Mournes,N10001007,Annalong,N08001002 +,,,,Binnian,N08001008 +,,,,Donard,N08001018 +,,,,Kilkeel,N08001025 +,,,,Lisnacree,N08001029 +,,,,Murlough,N08001032 +,,,,Tollymore,N08001039 +,,Slieve Croob,N10001005,Ballydugan,N08001004 +,,,,Ballyward,N08001006 +,,,,Castlewellan,N08001011 +,,,,Drumaness,N08001020 +,,,,Dundrum,N08001021 +,,Downpatrick,N10001002,Cathedral,N08001012 +,,,,Knocknashinna,N08001027 +,,,,Lecale,N08001028 +,,,,Quoile,N08001034 +,,,,Strangford,N08001038 +,,Rowallane,N10001004,Ballynahinch,N08001005 +,,,,Crossgar and Killyleagh,N08001013 +,,,,Derryboy,N08001016 +,,,,Kilmore,N08001026 +,,,,Saintfield,N08001036 +North Down and Ards,N09000011,Ards Peninsula,N10001101,Ballywalter,N08001106 +,,,,Carrowdore,N08001110 +,,,,Kircubbin,N08001128 +,,,,Loughries,N08001129 +,,,,Portaferry,N08001132 +,,,,Portavogie,N08001133 +,,Comber,N10001105,Ballygowan,N08001102 +,,,,Comber North,N08001113 +,,,,Comber South,N08001114 +,,,,Comber West,N08001115 +,,,,Killinchy,N08001127 +,,Newtownards,N10001107,Conway Square,N08001116 +,,,,Cronstown,N08001117 +,,,,Glen,N08001120 +,,,,Gregstown,N08001121 +,,,,Movilla,N08001131 +,,,,Scrabo,N08001136 +,,,,West Winds,N08001140 +,,Bangor East and Donaghadee,N10001103,Ballycrochan,N08001101 +,,,,Ballymagee,N08001105 +,,,,Donaghadee,N08001119 +,,,,Groomsport,N08001122 +,,,,Silverbirch,N08001137 +,,,,Warren,N08001139 +,,Bangor Central,N10001102,Ballygrainey,N08001103 +,,,,Ballyholme,N08001104 +,,,,Bloomfield,N08001107 +,,,,Broadway,N08001108 +,,,,Castle,N08001111 +,,,,Harbour,N08001123 +,,Bangor West,N10001104,Bryansburn,N08001109 +,,,,Kilcooley,N08001126 +,,,,Rathgael,N08001134 +,,,,Rathmore,N08001135 +,,,,Silverstream,N08001138 +,,Holywood and Clandeboye,N10001106,Clandeboye,N08001112 +,,,,Cultra,N08001118 +,,,,Helen’s Bay,N08001124 +,,,,Holywood,N08001125 +,,,,Loughview,N08001130 From 7f0f41e53a1f0ff3f2c0426af4317a015f1ec842 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Mon, 23 Nov 2015 11:39:17 +0000 Subject: [PATCH 04/22] Allow importing postcodes with no location Some rows in the ONSPD that have no location - those where the 12th column is a '9'. The existing importers for ni and gb postcodes automatically ignore these, but it can be useful to include them for existence checks even if we can't give more geographical information about them. We implement `location_available_for_row` to return `False` if the quality row is `'9'`, `True` otherwise. This lets us import rows that have no location if we want to. To keep the Code-Point Open import behaviour we provide an option `--allow-no-location-postcodes` to turn this on; without this option the importer will not import these rows. --- .../management/commands/mapit_UK_import_onspd.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_onspd.py b/mapit_gb/management/commands/mapit_UK_import_onspd.py index 9b474660..564aeed6 100644 --- a/mapit_gb/management/commands/mapit_UK_import_onspd.py +++ b/mapit_gb/management/commands/mapit_UK_import_onspd.py @@ -100,13 +100,23 @@ class Command(Command): default=False, help='Set if you want to import terminated postcodes' ), + make_option( + '--allow-no-location-postcodes', + action='store_true', + dest='include-no-location', + default=False, + help='Set if you want to import postcodes without location info (quality: 9)' + ), ) def pre_row(self, row, options): if row[4] and not options['include-terminated']: return False # Terminated postcode - if row[11] == '9': - return False # PO Box etc. + if not(self.location_available_for_row(row) or options['include-no-location']): + return False # go no further unless we want codes with no location if self.code[0:2] in ('GY', 'JE', 'IM', 'BT'): return False # NI and channel islands handled by other commands return True + + def location_available_for_row(self, row): + return row[11] != '9' # PO Box etc. From 8dafb129e3a44a3a425b50d66ef102dd8eb1b5e3 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Wed, 25 Nov 2015 09:57:32 +0000 Subject: [PATCH 05/22] Merge onspd and nspd_crown_dependencies importers Now that we can handle rows with no location we don't need a separate importer for crown-dependencies. We add a new `--crown-dependencies` option to the onspd importer that takes the following options: `include`, `exclude`, or `only`. The default is `exclude` which means we retain the previous behaviour of the onspd importer to only import GB postcodes and require a second run to import the crown dependency ones. Choosing `include` means we import all the GB + crown dependency postcodes in one pass (we still ignore the NI ones though). The `only` option means that we want to only import crown dependency postcodes - this gives us the behaviour of the old nspd_crown_dependencies importer, which is useful should you choose to import GB postcodes from the Code-Point Open dataset. Note that crown dependency postcodes (currently) have no location information in ONSPD and while the importer has an option for allowing postcodes with no location to be imported (--allow-no-location-postcodes) this option has no effect on the behaviour of importing crown dependency postcodes. Crown Dependency postcodes are imported solely based on the value of the --crown-dependencies option, as outlined above. This could be confusing, but the help text should make it clear. --- ...mapit_UK_import_nspd_crown_dependencies.py | 27 -------- .../commands/mapit_UK_import_onspd.py | 64 +++++++++++++++++-- 2 files changed, 58 insertions(+), 33 deletions(-) delete mode 100644 mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py diff --git a/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py b/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py deleted file mode 100644 index 63714d8d..00000000 --- a/mapit_gb/management/commands/mapit_UK_import_nspd_crown_dependencies.py +++ /dev/null @@ -1,27 +0,0 @@ -# This script is used to import Crown Dependency postcode information from the -# National Statistics Postcode Database. -# http://www.ons.gov.uk/about-statistics/geography/products/geog-products-postcode/nspd/ - -from mapit.management.commands.mapit_import_postal_codes import Command -from optparse import make_option - -class Command(Command): - help = 'Imports Crown Dependency postcodes from the NSPD' - args = '' - option_defaults = {'strip': True, 'location': False} - option_list = Command.option_list + ( - make_option( - '--allow-terminated-postcodes', - action='store_true', - dest='include-terminated', - default=False, - help='Set if you want to import terminated postcodes' - ), - ) - - def pre_row(self, row, options): - if row[4] and not options['include-terminated']: - return False # Terminated postcode - if self.code[0:2] not in ('GY', 'JE', 'IM'): - return False # Only importing Crown dependencies from NSPD - return True diff --git a/mapit_gb/management/commands/mapit_UK_import_onspd.py b/mapit_gb/management/commands/mapit_UK_import_onspd.py index 564aeed6..09432412 100644 --- a/mapit_gb/management/commands/mapit_UK_import_onspd.py +++ b/mapit_gb/management/commands/mapit_UK_import_onspd.py @@ -105,18 +105,70 @@ class Command(Command): action='store_true', dest='include-no-location', default=False, - help='Set if you want to import postcodes without location info (quality: 9)' + help='Set if you want to import postcodes without location info (quality: 9). Note that Crown Dependency postcodes have no location and if you choose to import these postcodes they will be imported regardless of your choice for this option.' + ), + make_option( + '--crown-dependencies', + action='store', + dest='crown-dependencies', + default='exclude', + help=('How to handle crown dependency postocdes. Set to "include" ' + 'to import them, set to "exclude" to ignore them, set to ' + '"only" to import only these. (Default: exclude). Note that ' + 'Crown Dependency postcodes have no location info and are ' + 'imported solely based on this option, regardless of the ' + 'presence of --allow-no-location-postcodes.') ), ) + def handle_label(self, file, **options): + # Check our crown-dependencies option is correct + if not options['crown-dependencies'] in ('include', 'exclude', 'only'): + raise RuntimeError('Invalid value for --crown-dependencies "%s" must be "include", "exclude", or "only". ' % options['crown-dependencies']) + self.process(file, options) + def pre_row(self, row, options): - if row[4] and not options['include-terminated']: + if self.northern_ireland(row): + return False # NI handled by other importer + elif self.reject_row_based_on_termination_data(row, options): return False # Terminated postcode - if not(self.location_available_for_row(row) or options['include-no-location']): + elif self.reject_row_based_on_location_data(row, options): return False # go no further unless we want codes with no location - if self.code[0:2] in ('GY', 'JE', 'IM', 'BT'): - return False # NI and channel islands handled by other commands - return True + elif self.reject_row_based_on_crown_dependency_data(row, options): + return False # handle crown depenency options + else: + return True def location_available_for_row(self, row): return row[11] != '9' # PO Box etc. + + def crown_dependency(self, _row): + return self.code[0:2] in ('GY', 'JE', 'IM') + + def northern_ireland(self, _row): + return self.code[0:2] == 'BT' + + def reject_row_based_on_location_data(self, row, options): + if self.location_available_for_row(row): + return False # don't reject rows with locations + elif self.allow_row_with_no_location(row, options): + return False # don't reject rows without locations if we allow them + else: + return True # no location and not allowed, reject + + def reject_row_based_on_termination_data(self, row, options): + return row[4] and not options['include-terminated'] + + def allow_row_with_no_location(self, row, options): + # crown dependencies have no location data in ONSPD so we allow the + # row and defer the decision to reject the row until we examine the + # 'crown-dependencies' option. + return options['include-no-location'] or self.crown_dependency(row) + + def reject_row_based_on_crown_dependency_data(self, row, options): + if self.crown_dependency_postcode(): + return options['crown-dependencies'] == 'exclude' # reject if we should exclude these codes + elif options['crown-dependencies'] == 'only': + return True # if we're only importing these codes, reject other codes + else: + return False # otherwise keep From 503745d4b593a0b9883c04bdbf351bf9902d1f87 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 15:59:43 +0000 Subject: [PATCH 06/22] Add command for importing osni boundary data This is mostly a rip from import_boundary_line but instead of taking one file as an argument we specify which files to work on as options because the OSNI releases are single files per shape, unlike Boundary-Line which has everything in one folder. We could expect a user to put everything in one folder first, but we need to handle each shapefile differently depending on how we expect it to be used. For example the Westminster Parliamentary constituencies boundaries have the name in PC_NAME and the gss code in PC_ID, whereas the LGD file has them in LGDNAME and LGDCode. We add a new codetype for identifying the OSNI id of the boundaries and a new nametype for the OSNI names. --- mapit_gb/fixtures/uk.json | 4 +- .../commands/mapit_UK_import_osni.py | 197 ++++++++++++++++++ 2 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 mapit_gb/management/commands/mapit_UK_import_osni.py diff --git a/mapit_gb/fixtures/uk.json b/mapit_gb/fixtures/uk.json index 62ab3975..494e1bf7 100644 --- a/mapit_gb/fixtures/uk.json +++ b/mapit_gb/fixtures/uk.json @@ -57,10 +57,12 @@ { "pk": 1, "model": "mapit.codetype", "fields": { "code": "ons", "description": "SNAC" } }, { "pk": 2, "model": "mapit.codetype", "fields": { "code": "gss", "description": "GSS (SNAC replacement)" } }, { "pk": 3, "model": "mapit.codetype", "fields": { "code": "unit_id", "description": "Boundary-Line (OS Admin Area ID)" } }, + { "pk": 4, "model": "mapit.codetype", "fields": { "code": "osni_oid", "description": "OSNI Object ID" } }, { "pk": 1, "model": "mapit.nametype", "fields": { "code": "O", "description": "Ordnance Survey" } }, { "pk": 2, "model": "mapit.nametype", "fields": { "code": "S", "description": "ONS (SNAC/GSS)" } }, - { "pk": 3, "model": "mapit.nametype", "fields": { "code": "M", "description": "Override name" } } + { "pk": 3, "model": "mapit.nametype", "fields": { "code": "M", "description": "Override name" } }, + { "pk": 4, "model": "mapit.nametype", "fields": { "code": "N", "description": "OSNI" } } ] diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py new file mode 100644 index 00000000..fffa9d68 --- /dev/null +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -0,0 +1,197 @@ +# coding=UTF-8 +# This script is used to import information from OSNI releases, +# which contains digital boundaries for administrative areas within +# Northern Ireland. + +import sys +import csv +import os +from optparse import make_option + +from django.core.management.base import NoArgsCommand +# Not using LayerMapping as want more control, but what it does is what this does +# from django.contrib.gis.utils import LayerMapping +from django.contrib.gis.gdal import DataSource +from django.utils import six + +from mapit.models import Area, Name, Generation, Country, Type, CodeType, NameType +from mapit.management.command_utils import save_polygons, fix_invalid_geos_geometry + + +class Command(NoArgsCommand): + help = 'Import OSNI releases' + option_list = NoArgsCommand.option_list + ( + make_option( + '--control', action='store', dest='control', + help='Refer to a Python module that can tell us what has changed'), + make_option('--commit', action='store_true', dest='commit', help='Actually update the database'), + make_option( + '--wmc', action='store', dest='wmc_file', + help= 'Name of OSNI shapefile that contains Westminister Parliamentary constituency boundary' + ), + make_option( + '--lgd', action='store', dest='lgd_file', + help='Name of OSNI shapefile that contains Council boundary information'), + ) + + ons_code_to_shape = {} + osni_object_id_to_shape = {} + + def handle_noargs(self, **options): + if not options['control']: + raise Exception("You must specify a control file") + __import__(options['control']) + control = sys.modules[options['control']] + + if all(options[x] is None for x in ['lgd_file', 'wmc_file']): + raise Exception("You must specify at least one of wmc, or lgd.") + + if options['lgd_file']: + self.process_file(options['lgd_file'], 'LGD', control, options) + + if options['wmc_file']: + self.process_file(options['wmc_file'], 'WMC', control, options) + + def process_file(self, filename, area_code, control, options): + code_version = CodeType.objects.get(code=control.code_version()) + name_type = NameType.objects.get(code='N') + code_type_osni = CodeType.objects.get(code='osni_oid') + + print(filename) + current_generation = Generation.objects.current() + new_generation = Generation.objects.new() + if not new_generation: + raise Exception("No new generation to be used for import!") + + ds = DataSource(filename) + layer = ds[0] + + if area_code not in self.area_code_to_feature_field: + raise Exception("Don't know how to extract features from %s files" % area_code) + + for feat in layer: + name, ons_code, osni_object_id = self.extract_fields_from_feature(feat, area_code) + if ons_code in self.ons_code_to_shape: + m, poly = self.ons_code_to_shape[ons_code] + try: + m_name = m.names.get(type=name_type).name + except Name.DoesNotExist: + m_name = m.name # If running without commit for dry run, so nothing being stored in db + if name != m_name: + raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) + # Otherwise, combine the two shapes for one area + poly.append(feat.geom) + continue + + if osni_object_id in self.osni_object_id_to_shape: + m, poly = self.osni_object_id_to_shape[osni_object_id] + try: + m_name = m.names.get(type=name_type).name + except Name.DoesNotExist: + m_name = m.name # If running without commit for dry run, so nothing being stored in db + if name != m_name: + raise Exception("OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name)) + # Otherwise, combine the two shapes for one area + poly.append(feat.geom) + continue + + country = 'N' + + try: + check = control.check(name, area_code, country, feat.geom) + if check is True: + raise Area.DoesNotExist + if isinstance(check, Area): + m = check + ons_code = m.codes.get(type=code_version).code + elif ons_code: + m = Area.objects.get(codes__type=code_version, codes__code=ons_code) + elif osni_object_id: + m = Area.objects.get( + codes__type=code_type_osni, codes__code=osni_object_id, + generation_high=current_generation + ) + m_name = m.names.get(type=name_type).name + if name != m_name: + raise Exception( + "OSNI Object ID code %s is %s in DB but %s in SHP file" % + (osni_object_id, m_name, name) + ) + else: + raise Exception( + 'Area "%s" (%s) has neither ONS code nor OSNI Object ID' % + (name, area_code) + ) + if int(options['verbosity']) > 1: + print(" Area matched, %s" % (m, )) + except Area.DoesNotExist: + print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name)) + m = Area( + name=name, # If committing, this will be overwritten by the m.names.update_or_create + type=Type.objects.get(code=area_code), + country=Country.objects.get(code=country), + generation_low=new_generation, + generation_high=new_generation, + ) + + if m.generation_high and current_generation and m.generation_high.id < current_generation.id: + raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) + m.generation_high = new_generation + if options['commit']: + m.save() + + # Make a GEOS geometry only to check for validity: + g = feat.geom + geos_g = g.geos + if not geos_g.valid: + print(" Geometry of %s %s not valid" % (ons_code, m)) + geos_g = fix_invalid_geos_geometry(geos_g) + if geos_g is None: + raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) + g = None + else: + g = geos_g.ogr + + poly = [g] + + if options['commit']: + m.names.update_or_create(type=name_type, defaults={'name': name}) + if ons_code: + self.ons_code_to_shape[ons_code] = (m, poly) + if options['commit']: + m.codes.update_or_create(type=code_version, defaults={'code': ons_code}) + if osni_object_id: + self.osni_object_id_to_shape[osni_object_id] = (m, poly) + if options['commit']: + m.codes.update_or_create(type=code_type_osni, defaults={'code': osni_object_id}) + + if options['commit']: + save_polygons(self.osni_object_id_to_shape) + save_polygons(self.ons_code_to_shape) + + def extract_fields_from_feature(self, feature, area_code): + name = self.extract_field_from_feature(feature, area_code, 'name') + name = self.format_name(name) + + ons_code = self.extract_field_from_feature(feature, area_code, 'ons_code') + + osni_object_id = self.extract_field_from_feature(feature, area_code, 'osni_object_id') + osni_object_id = "%s-%s" % (area_code, str(osni_object_id)) + + return (name, ons_code, osni_object_id) + + def extract_field_from_feature(self, feature, area_code, field): + if field in self.area_code_to_feature_field[area_code]: + self.area_code_to_feature_field[area_code][field].value + else: + return None + + area_code_to_feature_field = { + 'WMC': {'name': 'PC_NAME', 'ons_code': 'PC_ID', 'osni_object_id': 'OBJECTID'}, + 'LGD': {'name': 'LGDNAME', 'ons_code': 'LGDCode', 'osni_object_id': 'OBJECTID'}, + } + + def format_name(self, name): + if not isinstance(name, six.text_type): + name = name.decode('iso-8859-1') + return name From d861f29a92d23c8a58889e49205a924f823ff043 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 16:05:58 +0000 Subject: [PATCH 07/22] Add option for importing OSNI ward boundaries Name is in WARDNAME and GSS code is in WardCode. We did see them publish this data in a set that had no WardCode, but did have LGDName so we could use our ni-electoral-areas-2015.csv fixture to do a name match to get the GSS code. Hopefully though they won't publish that dataset again. --- mapit_gb/management/commands/mapit_UK_import_osni.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index fffa9d68..bc9ca434 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -25,6 +25,10 @@ class Command(NoArgsCommand): '--control', action='store', dest='control', help='Refer to a Python module that can tell us what has changed'), make_option('--commit', action='store_true', dest='commit', help='Actually update the database'), + make_option( + '--lgw', action='store', dest='lgw_file', + help='Name of OSNI shapefile that contains Ward boundary information' + ), make_option( '--wmc', action='store', dest='wmc_file', help= 'Name of OSNI shapefile that contains Westminister Parliamentary constituency boundary' @@ -43,8 +47,11 @@ def handle_noargs(self, **options): __import__(options['control']) control = sys.modules[options['control']] - if all(options[x] is None for x in ['lgd_file', 'wmc_file']): - raise Exception("You must specify at least one of wmc, or lgd.") + if all(options[x] is None for x in ['lgw_file', 'lgd_file', 'wmc_file']): + raise Exception("You must specify at least one of lgw, wmc, or lgd.") + + if options['lgw_file']: + self.process_file(options['lgw_file'], 'LGW', control, options) if options['lgd_file']: self.process_file(options['lgd_file'], 'LGD', control, options) @@ -189,6 +196,7 @@ def extract_field_from_feature(self, feature, area_code, field): area_code_to_feature_field = { 'WMC': {'name': 'PC_NAME', 'ons_code': 'PC_ID', 'osni_object_id': 'OBJECTID'}, 'LGD': {'name': 'LGDNAME', 'ons_code': 'LGDCode', 'osni_object_id': 'OBJECTID'}, + 'LGW': {'name': 'WARDNAME', 'ons_code': 'WardCode', 'osni_object_id': 'OBJECTID'}, } def format_name(self, name): From 1f4b13ad1a8630ac095f2ac01cd3567ff2ebfe62 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 16:09:25 +0000 Subject: [PATCH 08/22] Import Northern Ireland Assembly constituencies We process the Westminister file twice, once to generate areas with type WMC and then again to generate areas with type NIE. This is what the old shape-less importer did so I assume it's still good. These NIE areas don't have GSS codes because they are legally identical to the Westminister constituencies and don't have a code issued by the ONS. --- mapit_gb/management/commands/mapit_UK_import_osni.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index bc9ca434..c7b103ef 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -31,7 +31,10 @@ class Command(NoArgsCommand): ), make_option( '--wmc', action='store', dest='wmc_file', - help= 'Name of OSNI shapefile that contains Westminister Parliamentary constituency boundary' + help=( + 'Name of OSNI shapefile that contains Westminister Parliamentary constituency boundary' + 'information (also used for Northern Ireland Assembly constituencies)' + ) ), make_option( '--lgd', action='store', dest='lgd_file', @@ -58,6 +61,7 @@ def handle_noargs(self, **options): if options['wmc_file']: self.process_file(options['wmc_file'], 'WMC', control, options) + self.process_file(options['wmc_file'], 'NIE', control, options) def process_file(self, filename, area_code, control, options): code_version = CodeType.objects.get(code=control.code_version()) @@ -195,6 +199,9 @@ def extract_field_from_feature(self, feature, area_code, field): area_code_to_feature_field = { 'WMC': {'name': 'PC_NAME', 'ons_code': 'PC_ID', 'osni_object_id': 'OBJECTID'}, + # We don't have GSS codes for NIE areas, because we generate them from + # the WMC data and can't have duplicates + 'NIE': {'name': 'PC_NAME', 'osni_object_id': 'OBJECTID'}, 'LGD': {'name': 'LGDNAME', 'ons_code': 'LGDCode', 'osni_object_id': 'OBJECTID'}, 'LGW': {'name': 'WARDNAME', 'ons_code': 'WardCode', 'osni_object_id': 'OBJECTID'}, } From b861d8f8bce8e7e9a0ad90b56840dd951387d740 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 16:11:34 +0000 Subject: [PATCH 09/22] Import NI Electoral Areas (LGE) These are the areas that live between LGD (NI Councils) and LGW (NI Wards). The shapefile released by the OSNI does not include GSS codes so we have to synthesize them by doing a name lookup against the ni-electoral-areas-2015.csv fixture. We've asked OSNI if they plan to expose GSS codes in this dataset. --- .../commands/mapit_UK_import_osni.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index c7b103ef..8cc497d4 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -39,6 +39,9 @@ class Command(NoArgsCommand): make_option( '--lgd', action='store', dest='lgd_file', help='Name of OSNI shapefile that contains Council boundary information'), + make_option( + '--lge', action='store', dest='lge_file', + help='Name of OSNI shapefile that contains Electoral Area boundary information'), ) ons_code_to_shape = {} @@ -50,12 +53,15 @@ def handle_noargs(self, **options): __import__(options['control']) control = sys.modules[options['control']] - if all(options[x] is None for x in ['lgw_file', 'lgd_file', 'wmc_file']): - raise Exception("You must specify at least one of lgw, wmc, or lgd.") + if all(options[x] is None for x in ['lgw_file', 'lgd_file', 'lge_file', 'wmc_file']): + raise Exception("You must specify at least one of lgw, wmc, lgd, or lge.") if options['lgw_file']: self.process_file(options['lgw_file'], 'LGW', control, options) + if options['lge_file']: + self.process_file(options['lge_file'], 'LGE', control, options) + if options['lgd_file']: self.process_file(options['lgd_file'], 'LGD', control, options) @@ -193,7 +199,23 @@ def extract_fields_from_feature(self, feature, area_code): def extract_field_from_feature(self, feature, area_code, field): if field in self.area_code_to_feature_field[area_code]: - self.area_code_to_feature_field[area_code][field].value + field_extractor = self.area_code_to_feature_field[area_code][field] + if hasattr(field_extractor, '__call__'): + return field_extractor(self, feature, area_code) + else: + return feature[field_extractor].value + else: + return None + + lge_ons_codes = {} + + def extract_lge_ons_code_from_fixture(self, feature, _area_code): + if not self.lge_ons_codes: + self.populate_osni_missing_ons_codes() + + lge_name = self.format_name(self.extract_field_from_feature(feature, 'LGE', 'name')) + if lge_name in self.lge_ons_codes: + return self.lge_ons_codes[lge_name] else: return None @@ -204,8 +226,20 @@ def extract_field_from_feature(self, feature, area_code, field): 'NIE': {'name': 'PC_NAME', 'osni_object_id': 'OBJECTID'}, 'LGD': {'name': 'LGDNAME', 'ons_code': 'LGDCode', 'osni_object_id': 'OBJECTID'}, 'LGW': {'name': 'WARDNAME', 'ons_code': 'WardCode', 'osni_object_id': 'OBJECTID'}, + 'LGE': {'name': 'FinalR_DEA', 'ons_code': extract_lge_ons_code_from_fixture, 'osni_object_id': 'OBJECTID'} } + def populate_osni_missing_ons_codes(self): + ni_areas = csv.reader(open(os.path.dirname(__file__) + '/../../data/ni-electoral-areas-2015.csv')) + next(ni_areas) # comment line + next(ni_areas) # header row + for _lgd_name, _lgd_gss_code, lge_name, lge_gss_code, _lgw_name, _lgw_gss_code in ni_areas: + if not lge_name: + next + else: + if lge_name not in self.lge_ons_codes: + self.lge_ons_codes[lge_name] = lge_gss_code + def format_name(self, name): if not isinstance(name, six.text_type): name = name.decode('iso-8859-1') From 47a26c309c927765269a5b112f914716a7ecf3e2 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 16:15:13 +0000 Subject: [PATCH 10/22] Import NI EUR area Note that importing this area generates some warnings about invalid geometry, but the existing "fix_invalid_geos_geometry" is able to turn the data into something that it considers valid. --- .../management/commands/mapit_UK_import_osni.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index 8cc497d4..428c9267 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -42,6 +42,9 @@ class Command(NoArgsCommand): make_option( '--lge', action='store', dest='lge_file', help='Name of OSNI shapefile that contains Electoral Area boundary information'), + make_option( + '--eur', action='store', dest='eur_file', + help='Name of OSNI shapefile that contains European Region boundary information'), ) ons_code_to_shape = {} @@ -53,8 +56,8 @@ def handle_noargs(self, **options): __import__(options['control']) control = sys.modules[options['control']] - if all(options[x] is None for x in ['lgw_file', 'lgd_file', 'lge_file', 'wmc_file']): - raise Exception("You must specify at least one of lgw, wmc, lgd, or lge.") + if all(options[x] is None for x in ['lgw_file', 'lgd_file', 'lge_file', 'wmc_file', 'eur_file']): + raise Exception("You must specify at least one of lgw, wmc, lgd, lge, or eur.") if options['lgw_file']: self.process_file(options['lgw_file'], 'LGW', control, options) @@ -69,6 +72,9 @@ def handle_noargs(self, **options): self.process_file(options['wmc_file'], 'WMC', control, options) self.process_file(options['wmc_file'], 'NIE', control, options) + if options['eur_file']: + self.process_file(options['eur_file'], 'EUR', control, options) + def process_file(self, filename, area_code, control, options): code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code='N') @@ -219,7 +225,14 @@ def extract_lge_ons_code_from_fixture(self, feature, _area_code): else: return None + def ni_eur_name(self, _feature, _area_code): + return 'Northern Ireland' + + def ni_eur_ons_code(self, _feature, _area_code): + return 'N07000001' + area_code_to_feature_field = { + 'EUR': {'name': ni_eur_name, 'ons_code': ni_eur_ons_code, 'osni_object_id': 'OBJECTID'}, 'WMC': {'name': 'PC_NAME', 'ons_code': 'PC_ID', 'osni_object_id': 'OBJECTID'}, # We don't have GSS codes for NIE areas, because we generate them from # the WMC data and can't have duplicates From b924486e2c87f9365290e5c1da013984251eafb1 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 16:16:57 +0000 Subject: [PATCH 11/22] Add LGW->LGE and LGE->LGD parent lookups --- mapit_gb/management/commands/mapit_UK_find_parents.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mapit_gb/management/commands/mapit_UK_find_parents.py b/mapit_gb/management/commands/mapit_UK_find_parents.py index 45c82168..04090f9b 100644 --- a/mapit_gb/management/commands/mapit_UK_find_parents.py +++ b/mapit_gb/management/commands/mapit_UK_find_parents.py @@ -34,4 +34,8 @@ class Command(FindParentsCommand): # Scilly Isles 'CPC': ('DIS', 'UTA', 'MTD', 'LBO', 'COI'), 'CPW': 'CPC', + # A Northern Ireland ward's parent is a Northern Ireland electoral area + 'LGW': 'LGE', + # A Northern Ireland electoral area's parent is a Northern Ireland Council district + 'LGE': 'LGD', } From 5f19d5fd721bf87b724bdc80e8b112ff83fba9e2 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 16:39:49 +0000 Subject: [PATCH 12/22] Simplify field extraction by using objects not hashes This means we don't need to deal with strings vs callables and can isolate complexity to the area codes that need it (LGE and NIE). --- .../commands/mapit_UK_import_osni.py | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index 428c9267..c07a5bc4 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -79,6 +79,10 @@ def process_file(self, filename, area_code, control, options): code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code='N') code_type_osni = CodeType.objects.get(code='osni_oid') + if not hasattr(self, area_code): + raise Exception("Don't know how to extract features from %s files" % area_code) + + area_code_info = getattr(self, area_code)() print(filename) current_generation = Generation.objects.current() @@ -89,11 +93,9 @@ def process_file(self, filename, area_code, control, options): ds = DataSource(filename) layer = ds[0] - if area_code not in self.area_code_to_feature_field: - raise Exception("Don't know how to extract features from %s files" % area_code) - for feat in layer: - name, ons_code, osni_object_id = self.extract_fields_from_feature(feat, area_code) + name, ons_code, osni_object_id = area_code_info.extract_fields(feat) + name = self.format_name(name) if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: @@ -192,66 +194,64 @@ def process_file(self, filename, area_code, control, options): save_polygons(self.osni_object_id_to_shape) save_polygons(self.ons_code_to_shape) - def extract_fields_from_feature(self, feature, area_code): - name = self.extract_field_from_feature(feature, area_code, 'name') - name = self.format_name(name) - - ons_code = self.extract_field_from_feature(feature, area_code, 'ons_code') - - osni_object_id = self.extract_field_from_feature(feature, area_code, 'osni_object_id') - osni_object_id = "%s-%s" % (area_code, str(osni_object_id)) - - return (name, ons_code, osni_object_id) - - def extract_field_from_feature(self, feature, area_code, field): - if field in self.area_code_to_feature_field[area_code]: - field_extractor = self.area_code_to_feature_field[area_code][field] - if hasattr(field_extractor, '__call__'): - return field_extractor(self, feature, area_code) - else: - return feature[field_extractor].value - else: - return None - - lge_ons_codes = {} - - def extract_lge_ons_code_from_fixture(self, feature, _area_code): - if not self.lge_ons_codes: - self.populate_osni_missing_ons_codes() - - lge_name = self.format_name(self.extract_field_from_feature(feature, 'LGE', 'name')) - if lge_name in self.lge_ons_codes: - return self.lge_ons_codes[lge_name] - else: - return None + class WMC: + def extract_fields(self, feature): + name = feature['PC_NAME'].value + ons_code = feature['PC_ID'].value + object_id = 'WMC-%s' % str(feature['OBJECTID'].value) + return (name, ons_code, object_id) - def ni_eur_name(self, _feature, _area_code): - return 'Northern Ireland' - - def ni_eur_ons_code(self, _feature, _area_code): - return 'N07000001' - - area_code_to_feature_field = { - 'EUR': {'name': ni_eur_name, 'ons_code': ni_eur_ons_code, 'osni_object_id': 'OBJECTID'}, - 'WMC': {'name': 'PC_NAME', 'ons_code': 'PC_ID', 'osni_object_id': 'OBJECTID'}, + class NIE: # We don't have GSS codes for NIE areas, because we generate them from # the WMC data and can't have duplicates - 'NIE': {'name': 'PC_NAME', 'osni_object_id': 'OBJECTID'}, - 'LGD': {'name': 'LGDNAME', 'ons_code': 'LGDCode', 'osni_object_id': 'OBJECTID'}, - 'LGW': {'name': 'WARDNAME', 'ons_code': 'WardCode', 'osni_object_id': 'OBJECTID'}, - 'LGE': {'name': 'FinalR_DEA', 'ons_code': extract_lge_ons_code_from_fixture, 'osni_object_id': 'OBJECTID'} - } - - def populate_osni_missing_ons_codes(self): - ni_areas = csv.reader(open(os.path.dirname(__file__) + '/../../data/ni-electoral-areas-2015.csv')) - next(ni_areas) # comment line - next(ni_areas) # header row - for _lgd_name, _lgd_gss_code, lge_name, lge_gss_code, _lgw_name, _lgw_gss_code in ni_areas: - if not lge_name: - next + def extract_fields(self, feature): + name = feature['PC_NAME'].value + object_id = 'NIE-%s' % str(feature['OBJECTID'].value) + return (name, None, object_id) + + class LGW: + def extract_fields(self, feature): + name = feature['WARDNAME'].value + ons_code = feature['WardCode'].value + object_id = 'LGW-%s' % str(feature['OBJECTID'].value) + return (name, ons_code, object_id) + + class LGD: + def extract_fields(self, feature): + name = feature['LGDNAME'].value + ons_code = feature['LGDCode'].value + object_id = 'LGD-%s' % str(feature['OBJECTID'].value) + return (name, ons_code, object_id) + + class LGE: + def __init__(self): + self.lge_ons_codes = {} + self._populate_osni_missing_ons_codes() + + def _populate_osni_missing_ons_codes(self): + ni_areas = csv.reader(open(os.path.dirname(__file__) + '/../../data/ni-electoral-areas-2015.csv')) + next(ni_areas) # comment line + next(ni_areas) # header row + for _lgd_name, _lgd_gss_code, lge_name, lge_gss_code, _lgw_name, _lgw_gss_code in ni_areas: + if not lge_name: + next + else: + if lge_name not in self.lge_ons_codes: + self.lge_ons_codes[lge_name] = lge_gss_code + + def extract_fields(self, feature): + name = feature['FinalR_DEA'].value + if name in self.lge_ons_codes: + ons_code = self.lge_ons_codes[name] else: - if lge_name not in self.lge_ons_codes: - self.lge_ons_codes[lge_name] = lge_gss_code + ons_code = None + object_id = 'LGE-%s' % str(feature['OBJECTID'].value) + return (name, ons_code, object_id) + + class EUR: + def extract_fields(self, feature): + object_id = 'EUR-%s' % str(feature['OBJECTID'].value) + return ('Northern Ireland', 'N07000001', object_id) def format_name(self, name): if not isinstance(name, six.text_type): From 645a722be9fe3600d3f007a9b85f933fdad8125a Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Thu, 10 Dec 2015 17:13:34 +0000 Subject: [PATCH 13/22] Allow specifying SRID of OSNI imports From looking at the shapefiles in a viewing tool (qgis) it appears that the co-ords are in the NI projection (29902) in some files, but in the 102100 project in others. For some files this matches up with the arcgis metadata pointed to by the OSNI for each release (e.g. the LGEs dataset[1] points to the following metadata as its source[2] which lists 29902 as the projection, and the LGW dataset[3] points to source metadata[4] which lists 102100 projection). For others however this is not true (e.g. the LGDs dataset[5] has a source metadata[6] that says 299002, but the downloaded shapefile is actually in 102100). As it appears this is changeable per release (possibly per download) we allow for telling the importer what srid a given file is in. If it's not 29902 we convert it to 29902 before importing so that everything is consistent. Note that for some reason the geometry imported from the shapefiles does not contain an SRID for some reason so we have to set it even if we don't have to transform it. As a further wrinkle, PostGIS doesn't support 102100, but it is mathematically equivalent to 3857 which it does support. Unfortunately using that projection causes failures during for point-based lookup of parents, but if we use 4326 instead it works. Apparently 102100 and 4326 are both "web mercator" projections so are probably very similar (if not exactly mathematically equivalent). Interestingly opening a shapefile that is in 102100 in a viewing tool such as qgis reports it as 4326 whereas a 29902 reports as a custom projection that is identical in all but name to 29902. This suggests it's safe to use 4326 as a replacement for 102100. The defaults we set for the options are based on the SRIDs of the data files we've downloaded in Dec 2015 - they may change over time. [1]: http://osni.spatial-ni.opendata.arcgis.com/datasets/981a83027c0e4790891baadcfaa359a3_4 [2]: https://gisservices.spatialni.gov.uk/arcgisc/rest/services/OpenData/OSNIOpenData_LargescaleBoundaries/MapServer/4 [3]: http://osni.spatial-ni.opendata.arcgis.com/datasets/55cd419b2d2144de9565c9b8f73a226d_0 [4]: https://services3.arcgis.com/dNsInyVNGMqG1QjF/arcgis/rest/services/OSNI_Open_Data_Largescale_Boundaries_Wards_2012/FeatureServer/0 [5]: http://osni.spatial-ni.opendata.arcgis.com/datasets/a55726475f1b460c927d1816ffde6c72_2 [6]: https://gisservices.spatialni.gov.uk/arcgisc/rest/services/OpenData/OSNIOpenData_LargescaleBoundaries/MapServer/2 --- .../commands/mapit_UK_import_osni.py | 72 +++++++++++++++---- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index c07a5bc4..2f621232 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -45,6 +45,34 @@ class Command(NoArgsCommand): make_option( '--eur', action='store', dest='eur_file', help='Name of OSNI shapefile that contains European Region boundary information'), + + # OSNI datasets are exported in either 29902 or 102100 projections. + # PostGIS doesn't support 102100, but it is mathematically equivalent + # to 3857 which it does support. Unfortunately using that projection + # causes failures during for point-based lookup of parents, but if we + # use 4326 instead it works. Apparently 102100 and 4326 are both "web + # mercator" projections so are probably very similar (if not exactly + # mathematically equivalent). Interestingly opening a shapefile that is + # in 102100 in a viewing tool such as qgis reports it as 4326 whereas a + # 29902 reports as a custom projection that is identical in all but name + # to 29902. This suggests it's safe to use 4326 as a replacement for + # 102100. The defaults here are based on the srids of the data + # downloaded in Dec 2015 - they may change over time. + make_option( + '--lgw-srid', action='store', type='int', dest='lgw_srid', default=4326, + help='SRID of Ward boundary information shapefile (default 4326)'), + make_option( + '--wmc-srid', action='store', type='int', dest='wmc_srid', default=4326, + help='SRID of Westminister Parliamentery constituency boundary information shapefile (default 4326)'), + make_option( + '--lgd-srid', action='store', type='int', dest='lgd_srid', default=4326, + help='SRID of Council boundary information shapefile (default 4326)'), + make_option( + '--lge-srid', action='store', type='int', dest='lge_srid', default=29902, + help='SRID of Electoral Area boundary information shapefile (default 29902)'), + make_option( + '--eur-srid', action='store', type='int', dest='eur_srid', default=29902, + help='SRID of European Region boundary information shapefile (default 29902)'), ) ons_code_to_shape = {} @@ -60,29 +88,29 @@ def handle_noargs(self, **options): raise Exception("You must specify at least one of lgw, wmc, lgd, lge, or eur.") if options['lgw_file']: - self.process_file(options['lgw_file'], 'LGW', control, options) + self.process_file(options['lgw_file'], 'LGW', options['lgw_srid'], control, options) if options['lge_file']: - self.process_file(options['lge_file'], 'LGE', control, options) + self.process_file(options['lge_file'], 'LGE', options['lge_srid'], control, options) if options['lgd_file']: - self.process_file(options['lgd_file'], 'LGD', control, options) + self.process_file(options['lgd_file'], 'LGD', options['lgd_srid'], control, options) if options['wmc_file']: - self.process_file(options['wmc_file'], 'WMC', control, options) - self.process_file(options['wmc_file'], 'NIE', control, options) + self.process_file(options['wmc_file'], 'WMC', options['wmc_srid'], control, options) + self.process_file(options['wmc_file'], 'NIE', options['wmc_srid'], control, options) if options['eur_file']: - self.process_file(options['eur_file'], 'EUR', control, options) + self.process_file(options['eur_file'], 'EUR', options['eur_srid'], control, options) - def process_file(self, filename, area_code, control, options): + def process_file(self, filename, area_code, srid, control, options): code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code='N') code_type_osni = CodeType.objects.get(code='osni_oid') if not hasattr(self, area_code): raise Exception("Don't know how to extract features from %s files" % area_code) - area_code_info = getattr(self, area_code)() + area_code_info = getattr(self, area_code)(srid) print(filename) current_generation = Generation.objects.current() @@ -177,6 +205,7 @@ def process_file(self, filename, area_code, control, options): else: g = geos_g.ogr + g = area_code_info.transform_geom(g) poly = [g] if options['commit']: @@ -194,14 +223,26 @@ def process_file(self, filename, area_code, control, options): save_polygons(self.osni_object_id_to_shape) save_polygons(self.ons_code_to_shape) - class WMC: + class AreaCodeShapefileInterpreter(object): + def __init__(self, srid): + self.srid = srid + + # Transform all shapefile geometry to 29902 for consistency if it's + # not already in that projection + def transform_geom(self, geom): + geom.srid = self.srid + if not(self.srid == 29902): + geom.transform(29902) + return geom + + class WMC(AreaCodeShapefileInterpreter): def extract_fields(self, feature): name = feature['PC_NAME'].value ons_code = feature['PC_ID'].value object_id = 'WMC-%s' % str(feature['OBJECTID'].value) return (name, ons_code, object_id) - class NIE: + class NIE(AreaCodeShapefileInterpreter): # We don't have GSS codes for NIE areas, because we generate them from # the WMC data and can't have duplicates def extract_fields(self, feature): @@ -209,22 +250,23 @@ def extract_fields(self, feature): object_id = 'NIE-%s' % str(feature['OBJECTID'].value) return (name, None, object_id) - class LGW: + class LGW(AreaCodeShapefileInterpreter): def extract_fields(self, feature): name = feature['WARDNAME'].value ons_code = feature['WardCode'].value object_id = 'LGW-%s' % str(feature['OBJECTID'].value) return (name, ons_code, object_id) - class LGD: + class LGD(AreaCodeShapefileInterpreter): def extract_fields(self, feature): name = feature['LGDNAME'].value ons_code = feature['LGDCode'].value object_id = 'LGD-%s' % str(feature['OBJECTID'].value) return (name, ons_code, object_id) - class LGE: - def __init__(self): + class LGE(AreaCodeShapefileInterpreter): + def __init__(self, srid): + super(self.__class__, self).__init__(srid) self.lge_ons_codes = {} self._populate_osni_missing_ons_codes() @@ -248,7 +290,7 @@ def extract_fields(self, feature): object_id = 'LGE-%s' % str(feature['OBJECTID'].value) return (name, ons_code, object_id) - class EUR: + class EUR(AreaCodeShapefileInterpreter): def extract_fields(self, feature): object_id = 'EUR-%s' % str(feature['OBJECTID'].value) return ('Northern Ireland', 'N07000001', object_id) From eef599c596a0ff3d105e20a9b5d2f7178dfa1fa6 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Wed, 25 Nov 2015 13:32:12 +0000 Subject: [PATCH 14/22] Handle NI postcodes in ONSPD importer We add an option to allow importing the NI postcodes at the same time as the rest of the postcodes. The option takes 3 values: 'include', 'exclude', 'only' with the same behaviour as the --crown-dependencies option: * 'include' will import NI postcodes * 'exclude' will not import NI postcodes * 'only' will only import NI postcodes The default is 'exclude' to maintain previous behaviour. Unlike Crown Dependency postcodes NI postcodes might have location data, and so the --allow-no-location-postcodes setting (default false) does affect how we import NI postcodes. Setting both --crown-dependencies and --northern-ireland to 'only' is an error and will halt the importer before it begins. We've also updated the documentation provided by the options to be clearer about how the various options interact. Unlike the old nspd_ni importer that relied on the nspd_ni_areas importer to be run and the ni-electoral-areas.csv to directly assign areas to NI postcodes, this importer has no special handling. We assume that the new OSNI importer has been run and the relevant shapefiles have been imported, much like we assume that the boundary-line importer has been run to provide the areas for the rest of the UK. --- .../commands/mapit_UK_import_onspd.py | 67 +++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_onspd.py b/mapit_gb/management/commands/mapit_UK_import_onspd.py index 09432412..4f2744b4 100644 --- a/mapit_gb/management/commands/mapit_UK_import_onspd.py +++ b/mapit_gb/management/commands/mapit_UK_import_onspd.py @@ -89,7 +89,10 @@ class Command(Command): - help = 'Imports UK postcodes from the NSPD, excluding NI and Crown Dependencies' + help = ('Imports UK postcodes from the ONSPD.\n\nBy default imports only ' + 'live GB postcodes with a lat/lng, options are available to also ' + 'import terminated, NI, or Crown Dependency postcodes and those ' + 'without locations.') args = '' option_defaults = {'header-row': True, 'strip': True, 'srid': 27700, 'coord-field-lon': 10, 'coord-field-lat': 11} option_list = Command.option_list + ( @@ -98,14 +101,19 @@ class Command(Command): action='store_true', dest='include-terminated', default=False, - help='Set if you want to import terminated postcodes' + help=('Set if you want to import terminated postcodes. Affects all ' + 'postcodes: GB, NI, and Crown Dependencies') ), make_option( '--allow-no-location-postcodes', action='store_true', dest='include-no-location', default=False, - help='Set if you want to import postcodes without location info (quality: 9). Note that Crown Dependency postcodes have no location and if you choose to import these postcodes they will be imported regardless of your choice for this option.' + help=('Set if you want to import postcodes without location info ' + '(quality: 9). Affects GB and NI postcodes only. Crown ' + 'Dependency postcodes have no location and will be imported ' + 'based on the value of --crown-dependencies, regardless of ' + 'your choice for this option.') ), make_option( '--crown-dependencies', @@ -119,33 +127,56 @@ class Command(Command): 'imported solely based on this option, regardless of the ' 'presence of --allow-no-location-postcodes.') ), + make_option( + '--northern-ireland', + action='store', + dest='northern-ireland', + default='exclude', + help=('How to handle Northern Ireland postocdes. Set to "include" ' + 'to import them, set to "exclude" to ignore them, set to ' + '"only" to import only these. (Default: exclude). You should' + ' run mapit_UK_import_onspd_ni_areas before trying to import ' + 'any NI postcodes.') + ), ) def handle_label(self, file, **options): - # Check our crown-dependencies option is correct - if not options['crown-dependencies'] in ('include', 'exclude', 'only'): - raise RuntimeError('Invalid value for --crown-dependencies "%s" must be "include", "exclude", or "only". ' % options['crown-dependencies']) + self.check_options_are_valid(options) self.process(file, options) def pre_row(self, row, options): - if self.northern_ireland(row): - return False # NI handled by other importer - elif self.reject_row_based_on_termination_data(row, options): + if self.reject_row_based_on_termination_data(row, options): return False # Terminated postcode elif self.reject_row_based_on_location_data(row, options): return False # go no further unless we want codes with no location elif self.reject_row_based_on_crown_dependency_data(row, options): return False # handle crown depenency options - else: - return True + elif self.reject_row_based_on_northern_ireland_data(row, options): + return False # handle northern ireland options + + return True + + def check_options_are_valid(self, options): + # Check our crown-dependencies option is valid + if not options['crown-dependencies'] in ('include', 'exclude', 'only'): + raise RuntimeError(('Invalid value for --crown-dependencies "%s" must' + ' be "include", "exclude", or "only".') % options['crown-dependencies']) + # Check our northern-ireland option is valid + if not options['northern-ireland'] in ('include', 'exclude', 'only'): + raise RuntimeError(('Invalid value for --northern-ireland "%s" must ' + 'be "include", "exclude", or "only".') % options['northern-ireland']) + # Check we're not trying to "only" import both + if (options['crown-dependencies'] == 'only') and (options['northern-ireland'] == 'only'): + raise RuntimeError(('Cannot support "only" as value for both ' + '--northern-ireland and --crown-dependencies')) def location_available_for_row(self, row): return row[11] != '9' # PO Box etc. - def crown_dependency(self, _row): + def crown_dependency_postcode(self): return self.code[0:2] in ('GY', 'JE', 'IM') - def northern_ireland(self, _row): + def northern_ireland_postcode(self): return self.code[0:2] == 'BT' def reject_row_based_on_location_data(self, row, options): @@ -163,7 +194,7 @@ def allow_row_with_no_location(self, row, options): # crown dependencies have no location data in ONSPD so we allow the # row and defer the decision to reject the row until we examine the # 'crown-dependencies' option. - return options['include-no-location'] or self.crown_dependency(row) + return options['include-no-location'] or self.crown_dependency_postcode() def reject_row_based_on_crown_dependency_data(self, row, options): if self.crown_dependency_postcode(): @@ -172,3 +203,11 @@ def reject_row_based_on_crown_dependency_data(self, row, options): return True # if we're only importing these codes, reject other codes else: return False # otherwise keep + + def reject_row_based_on_northern_ireland_data(self, row, options): + if self.northern_ireland_postcode(): + return options['northern-ireland'] == 'exclude' # reject if we should exclude these codes + elif options['northern-ireland'] == 'only': + return True # if we're only importing these codes, reject other codes + else: + return False # otherwise keep From 91ca8f1bfcec6962198378fe52b28e1a176f0ebc Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Wed, 25 Nov 2015 16:48:39 +0000 Subject: [PATCH 15/22] Allow specifying srid for GB vs. NI postcodes The --gb-srid and --ni-srid options have defaults (27700 and 29902 respectively) that are sensible and will change the --srid option on a per row basis if the postcode is for Northern Ireland (e.g. starts with BT) or not. --- .../commands/mapit_UK_import_onspd.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mapit_gb/management/commands/mapit_UK_import_onspd.py b/mapit_gb/management/commands/mapit_UK_import_onspd.py index 4f2744b4..e313aa7d 100644 --- a/mapit_gb/management/commands/mapit_UK_import_onspd.py +++ b/mapit_gb/management/commands/mapit_UK_import_onspd.py @@ -138,6 +138,22 @@ class Command(Command): ' run mapit_UK_import_onspd_ni_areas before trying to import ' 'any NI postcodes.') ), + make_option( + '--gb-srid', + action='store', + dest='gb-srid', + default=27700, + help=('SRID for GB & Crown Dependency postcodes. Overrides --srid ' + 'value. (Default: 27700).') + ), + make_option( + '--ni-srid', + action='store', + dest='ni-srid', + default=29902, + help=('SRID for NI postcodes. Overrides --srid value for. (Default:' + ' 29902).') + ), ) def handle_label(self, file, **options): @@ -154,6 +170,11 @@ def pre_row(self, row, options): elif self.reject_row_based_on_northern_ireland_data(row, options): return False # handle northern ireland options + if self.northern_ireland_postcode(): + options['srid'] = options['ni-srid'] + else: + options['srid'] = options['gb-srid'] + return True def check_options_are_valid(self, options): From c55e7f60f57bb15eedccd216bfefec3b16c42840 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Tue, 24 Nov 2015 16:17:04 +0000 Subject: [PATCH 16/22] Provide ONSPD version of scilly command One command can, by checking the lengths of the rows, work on both Code-Point Open and ONSPD files for dealing with scilly wards. --- .../management/commands/mapit_UK_scilly.py | 68 +++++++++++++++---- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_scilly.py b/mapit_gb/management/commands/mapit_UK_scilly.py index 6bd67ee9..507b4aa8 100644 --- a/mapit_gb/management/commands/mapit_UK_scilly.py +++ b/mapit_gb/management/commands/mapit_UK_scilly.py @@ -5,13 +5,33 @@ import csv import re +from optparse import make_option from django.core.management.base import LabelCommand from mapit.models import Postcode, Area, Country, Type, CodeType, NameType class Command(LabelCommand): help = 'Sort out the Isles of Scilly' - args = '' + args = ' or ' + option_list = LabelCommand.option_list + ( + make_option( + '--allow-terminated-postcodes', + action='store_true', + dest='include-terminated', + default=False, + help=('Set if you want to fix wards for terminated postcodes (only ' + 'relevant for ONSPD files, Code-Point Open contains no ' + 'terminated postcodes)') + ), + make_option( + '--allow-no-location-postcodes', + action='store_true', + dest='include-no-location', + default=False, + help=('Set if you want to fix wards for postcodes with no location ' + '(quality 9 in ONSPD, quality 90 in Code Point Open)') + ), + ) def handle_label(self, file, **options): # The Isles of Scilly have changed their code in B-L, but Code-Point still has the old code currently @@ -47,17 +67,39 @@ def handle_label(self, file, **options): ward[new_ward_code] = area for row in csv.reader(open(file)): - if row[1] == '90': - continue - postcode = row[0].strip().replace(' ', '') - if len(row) == 10: - ons_code = row[9] - if not re.match('^E0500832[2-6]$', ons_code): - continue - else: - ons_code = ''.join(row[15:18]) - if ons_code[0:4] != '00HF': - continue + postcode, ward_code, lacks_location, terminated = self.extract_data(row) + if postcode[0:2] not in ('TR'): + continue # Ignore non scilly postcodes + if ward_code is None: + continue # Ignore if we couldn't extract a scilly ward_code + if terminated and not options['include-terminated']: + continue # Ignore terminated postcodes + if lacks_location and not options['include-no-location']: + continue # Ignore postcodes without a known location + pc = Postcode.objects.get(postcode=postcode) - pc.areas.add(ward[ons_code]) + pc.areas.add(ward[ward_code]) print(".", end=' ') + + def extract_data(self, row): + postcode = row[0].strip().replace(' ', '') + if len(row) == 10: # Post Aug 2011 Code-Point Open file + ward_code = row[9] if self.is_scilly_gss_code(row[9]) else None + lacks_location = row[1] == '90' + terminated = False + elif len(row) == 19: # Pre-Aug 2011 Code-Point Open file + code = ''.join(row[15:18]) + ward_code = code if self.is_scilly_ons_code(code) else None + lacks_location = row[1] == '90' + terminated = False + else: # ONSPD file + ward_code = row[7] if self.is_scilly_gss_code(row[7]) else None + lacks_location = row[11] == '9' + terminated = row[4] + return (postcode, ward_code, lacks_location, terminated) + + def is_scilly_gss_code(self, gss_code): + return re.match('^E0500832[2-6]$', gss_code) + + def is_scilly_ons_code(self, ons_code): + return ons_code[0:4] == '00HF' From 822f44336318bcbe9c29069fd9ade360eed3c672 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 18 Dec 2015 10:18:17 +0000 Subject: [PATCH 17/22] Add script for adding GSS codes to NI Areas Some mapit installations already have NI Areas with or without boundaries but these areas may not have GSS codes. This script uses the ni-electoral-areas-2015.csv hierarchy to find LGDs, LGEs, and LGWs by name and add their GSS codes. Because names are not neccessarily unique it respects the hierarchy in the fixture. If names cannot be found (it does a case-insensitive lookup) the row is ignored and a warning issued. --- .../mapit_UK_add_gss_codes_to_ni_areas.py | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py diff --git a/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py b/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py new file mode 100644 index 00000000..b343a8b7 --- /dev/null +++ b/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py @@ -0,0 +1,136 @@ +# coding=UTF-8 +# This script is used to add GSS codes to NI areas that don't have them, having +# been imported prior to OSNI releases. + +import csv +import os.path + +from django.core.management.base import NoArgsCommand +from mapit.models import Area, Generation, Country, CodeType +from django.utils import six + + +class Command(NoArgsCommand): + help = 'Uses fixtures to find NI areas and add any missing GSS codes to them' + + def handle_noargs(self, **options): + self.current_generation = Generation.objects.current() + self.new_generation = Generation.objects.new() + self.country = Country.objects.get(code='N') + if not self.new_generation: + raise Exception("No new generation to be used for import!") + + self.gss_code_type = CodeType.objects.get(code='gss') + + self.add_gss_code_to_eur_area() + council_areas = self.fetch_council_areas_hierarchy() + self.update_lgds(council_areas) + + def add_gss_code_to_eur_area(self): + euro_area, code_created = self.fetch_and_update_area( + Area.objects, 'EUR', 'Northern Ireland', 'N07000001' + ) + if euro_area: + self.report_result('EUR "Northern Ireland" - GSS code "N07000001"', code_created) + else: + print 'WARNING: No EUR area with name "Northern Ireland" to add GSS code "N07000001" to' + + def update_lgds(self, districts): + for district in districts: + district_name, district_gss_code = district + lgd, code_created = self.fetch_and_update_area( + Area.objects, 'LGD', district_name, district_gss_code + ) + if lgd: + self.report_result( + 'LGD "%s" - GSS code "%s" ' % (district_name, district_gss_code), + code_created) + self.update_lges_for_lgd(districts[district], lgd) + else: + print 'WARNING: No LGD with name "%s" to add GSS "%s" to' % (district_name, district_gss_code) + + def update_lges_for_lgd(self, electoral_areas, lgd): + for electoral_area in electoral_areas: + electoral_area_name, electoral_area_gss_code = electoral_area + lge, code_created = self.fetch_and_update_area( + lgd.children, 'LGE', electoral_area_name, electoral_area_gss_code + ) + if lge: + self.report_result( + ('LGE "%s" (child of LGD "%s") - GSS code "%s" ') + % (electoral_area_name, lgd.name, electoral_area_gss_code), + code_created) + self.update_lgws_for_lge(electoral_areas[electoral_area], lgd, lge) + else: + print ('WARNING: No LGE with name "%s" as child of LGD "%s" to ' + 'add GSS "%s" to') % (electoral_area_name, lgd.name, electoral_area_gss_code) + + def update_lgws_for_lge(self, wards, lgd, lge): + for ward in wards: + ward_name, ward_gss_code = ward + lgw, code_created = self.fetch_and_update_area( + lge.children, 'LGW', ward_name, ward_gss_code + ) + if lgw: + self.report_result( + ('LGW "%s" (child of LGE "%s" and LGD "%s") ' + '- GSS code "%s" ') % (ward_name, lge.name, lgd.name, ward_gss_code), + code_created) + else: + print ('WARNING: No LGW with name "%s" as child of LGE "%s" and' + ' LGD "%s" to add GSS "%s" to') % (ward_name, lge.name, lgd.name, ward_gss_code) + + def fetch_and_update_area(self, area_source, area_type, area_name, area_gss_code): + try: + area = area_source.get( + country=self.country, type__code=area_type, + names__name__iexact=area_name, + generation_low__lte=self.current_generation, + generation_high__gte=self.current_generation + ) + area.generation_high = self.new_generation + area.save() + _code, code_created = area.codes.get_or_create(type=self.gss_code_type, code=area_gss_code) + return area, code_created + except Area.DoesNotExist: + return None, False + + def report_result(self, message, code_created): + print message, + if code_created: + print "added" + else: + print "already present" + + def fetch_council_areas_hierarchy(self): + # Read in district + gss -> electoral area + gss -> ward +gss + ni_areas = csv.reader(open(os.path.dirname(__file__) + '/../../data/ni-electoral-areas-2015.csv')) + next(ni_areas) # comment line + next(ni_areas) # header row + + council_areas = {} + current_district = None + current_electoral_area = None + + for district, district_gss_code, electoral_area, electoral_area_gss_code, ward, ward_gss_code in ni_areas: + if not district: + district, district_gss_code = current_district + if not electoral_area: + electoral_area, electoral_area_gss_code = current_electoral_area + current_district = (self.format_name(district), district_gss_code) + current_electoral_area = (self.format_name(electoral_area), electoral_area_gss_code) + + if current_district not in council_areas: + council_areas[current_district] = {} + + if current_electoral_area not in council_areas[current_district]: + council_areas[current_district][current_electoral_area] = [] + + council_areas[current_district][current_electoral_area].append((self.format_name(ward), ward_gss_code)) + + return council_areas + + def format_name(self, name): + if not isinstance(name, six.text_type): + name = name.decode('utf-8 ') + return name From 26174206e1e41e4b591e6be83762e70cd1643fdd Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 18 Dec 2015 11:05:30 +0000 Subject: [PATCH 18/22] Add script for adding names to NI Areas The names in the OSNI data don't always match the names for the same areas in the ni-electoral-areas-2015.csv fixture which was extracted from the legislation. In some cases it's just an uppercase difference, or a lack of punctuation. In others the names are completely different. For example in the fixture GSS N09000011 is called "North Down and Ards" but in the OSNI shapefile it is called "East Coast". Turns out this is because the council voted to change the name to the OSNI one, but backed down after outcry and reverted[1]. This script goes through the fixture and matches on GSS code to find the Areas and add a new override name to the area if the fixture name is not already present. [1]: http://www.belfasttelegraph.co.uk/news/northern-ireland/backlash-forces-council-to-ditch-new-east-coast-name-that-cost-thousands-30902221.html --- .../mapit_UK_add_names_to_ni_areas.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py diff --git a/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py b/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py new file mode 100644 index 00000000..f521e918 --- /dev/null +++ b/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py @@ -0,0 +1,81 @@ +# coding=UTF-8 +# This script is used to add names to NI areas where the names of the areas +# don't include the names from the legislation derived fixture data. + +import csv +import os.path + +from django.core.management.base import NoArgsCommand +from mapit.models import Area, Generation, Country, NameType +from django.utils import six + + +class Command(NoArgsCommand): + help = 'Uses fixtures to find NI areas and add any missing names to them' + + def handle_noargs(self, **options): + self.current_generation = Generation.objects.current() + self.new_generation = Generation.objects.new() + self.country = Country.objects.get(code='N') + if not self.new_generation: + raise Exception("No new generation to be used for import!") + + self.name_type = NameType.objects.get(code='M') + + council_areas = self.fetch_council_areas() + self.update_area_names(council_areas) + + def update_area_names(self, council_areas): + for gss_code in council_areas: + name, area_type = council_areas[gss_code] + area, name_created = self.fetch_and_update_area(area_type, name, gss_code) + if area: + self.report_result('%s "%s" - name "%s" ' % (area_type, gss_code, name), name_created) + else: + print 'WARNING: No %s with GSS code "%s" to add name "%s" to' % (area_type, gss_code, name) + + def fetch_and_update_area(self, area_type, area_name, area_gss_code): + try: + area = Area.objects.get( + country=self.country, type__code=area_type, + codes__type__code='gss', codes__code=area_gss_code, + generation_low__lte=self.current_generation, + generation_high__gte=self.current_generation + ) + area.generation_high = self.new_generation + area.save() + _name, name_created = area.names.get_or_create(name=area_name, defaults={'type': self.name_type}) + return area, name_created + except Area.DoesNotExist: + return None, False + + def report_result(self, message, name_created): + print message, + if name_created: + print "added" + else: + print "already present" + + def fetch_council_areas(self): + # Read in district + gss -> electoral area + gss -> ward +gss + ni_areas = csv.reader(open(os.path.dirname(__file__) + '/../../data/ni-electoral-areas-2015.csv')) + next(ni_areas) # comment line + next(ni_areas) # header row + + council_areas = {} + + for district, district_gss_code, electoral_area, electoral_area_gss_code, ward, ward_gss_code in ni_areas: + if district: + council_areas[district_gss_code] = (self.format_name(district), 'LGD') + + if electoral_area: + council_areas[electoral_area_gss_code] = (self.format_name(electoral_area), 'LGE') + + council_areas[ward_gss_code] = (self.format_name(ward), 'LGW') + + return council_areas + + def format_name(self, name): + if not isinstance(name, six.text_type): + name = name.decode('utf-8 ') + return name From 99847c4346bbd026148b12fa505d1563e012ff6b Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 18 Dec 2015 14:56:08 +0000 Subject: [PATCH 19/22] Correct LGD names in ni-electoral-areas-2015.csv Mostly this is just extending the name to include the council type (District, Borough, or City), similar to naming of some council areas in the rest of the UK. In the case of "North Down and Ards" we also rename to their final name choice of "Ards and North Down". For "Derry and Strabane" and "Armagh, Banbridge and Craigavon" we also include "City" in the appropriate place ("Derry City" and "Armagh City"). --- mapit_gb/data/ni-electoral-areas-2015.csv | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mapit_gb/data/ni-electoral-areas-2015.csv b/mapit_gb/data/ni-electoral-areas-2015.csv index 697f3c01..397bfe7a 100644 --- a/mapit_gb/data/ni-electoral-areas-2015.csv +++ b/mapit_gb/data/ni-electoral-areas-2015.csv @@ -1,6 +1,6 @@ # from http://www.legislation.gov.uk/uksi/2014/270/schedule/made,,,,, District,District GSS Code,Electoral area,Electoral area GSS code,Ward,Ward GSS code -Antrim and Newtownabbey,N09000001,Dunsilly,N10000104,Cranfield,N08000115 +Antrim and Newtownabbey Borough Council,N09000001,Dunsilly,N10000104,Cranfield,N08000115 ,,,,Parkgate,N08000129 ,,,,Randalstown,N08000130 ,,,,Shilvodan,N08000133 @@ -40,7 +40,7 @@ Antrim and Newtownabbey,N09000001,Dunsilly,N10000104,Cranfield,N08000115 ,,,,Glebe,N08000120 ,,,,Glengormley,N08000121 ,,,,Hightown,N08000123 -"Armagh, Banbridge and Craigavon",N09000002,Armagh,N10000201,Blackwatertown,N08000207 +"Armagh City, Banbridge and Craigavon Borough Council",N09000002,Armagh,N10000201,Blackwatertown,N08000207 ,,,,Cathedral,N08000210 ,,,,Demesne,N08000213 ,,,,Keady,N08000220 @@ -81,7 +81,7 @@ Antrim and Newtownabbey,N09000001,Dunsilly,N10000104,Cranfield,N08000115 ,,,,Gilford,N08000217 ,,,,Loughbrickland,N08000225 ,,,,Rathfriland,N08000234 -Belfast,N09000003,Castle,N10000304,Bellevue,N08000309 +Belfast City Council,N09000003,Castle,N10000304,Bellevue,N08000309 ,,,,Cavehill,N08000314 ,,,,Chichester Park,N08000316 ,,,,Duncairn,N08000322 @@ -141,7 +141,7 @@ Belfast,N09000003,Castle,N10000304,Bellevue,N08000309 ,,,,Connswater,N08000320 ,,,,Sydenham,N08000353 ,,,,Woodstock,N08000359 -Causeway Coast and Glens,N09000004,The Glens,N10000407,Ballycastle,N08000404 +Causeway Coast and Glens Borough Council,N09000004,The Glens,N10000407,Ballycastle,N08000404 ,,,,Kinbane,N08000425 ,,,,Loughguile and Stranocum,N08000426 ,,,,Lurigethan,N08000427 @@ -181,7 +181,7 @@ Causeway Coast and Glens,N09000004,The Glens,N10000407,Ballycastle,N08000404 ,,,,Greystone,N08000422 ,,,,Magilligan,N08000429 ,,,,Roeside,N08000435 -Derry and Strabane,N09000005,Ballyarnett,N10000501,Carn Hill,N08000505 +Derry City and Strabane District Council,N09000005,Ballyarnett,N10000501,Carn Hill,N08000505 ,,,,Culmore,N08000513 ,,,,Galliagh,N08000521 ,,,,Shantallow,N08000531 @@ -221,7 +221,7 @@ Derry and Strabane,N09000005,Ballyarnett,N10000501,Carn Hill,N08000505 ,,,,Glenderg,N08000522 ,,,,Newtownstewart,N08000528 ,,,,Sion Mills,N08000534 -Fermanagh and Omagh,N09000006,Erne West,N10000604,Belcoo and Garrison,N08000602 +Fermanagh and Omagh District Council,N09000006,Erne West,N10000604,Belcoo and Garrison,N08000602 ,,,,"Boho, Cleenish and Letterbreen",N08000605 ,,,,Derrygonnelly,N08000611 ,,,,Derrylin,N08000612 @@ -261,7 +261,7 @@ Fermanagh and Omagh,N09000006,Erne West,N10000604,Belcoo and Garrison,N08000602 ,,,,Maguiresbridge,N08000629 ,,,,Newtownbutler,N08000630 ,,,,Rosslea,N08000634 -Lisburn and Castlereagh,N09000007,Killultagh,N10000705,Ballinderry,N08000701 +Lisburn and Castlereagh City Council,N09000007,Killultagh,N10000705,Ballinderry,N08000701 ,,,,Glenavy,N08000718 ,,,,Maghaberry,N08000730 ,,,,Stonyford,N08000738 @@ -301,7 +301,7 @@ Lisburn and Castlereagh,N09000007,Killultagh,N10000705,Ballinderry,N08000701 ,,,,Enler,N08000716 ,,,,Graham’s Bridge,N08000719 ,,,,Moneyreagh,N08000734 -Mid and East Antrim,N09000008,Knockagh,N10000806,Burleigh Hill,N08000810 +Mid and East Antrim Borough Council,N09000008,Knockagh,N10000806,Burleigh Hill,N08000810 ,,,,Gortalee,N08000823 ,,,,Greenisland,N08000825 ,,,,Sunnylands,N08000836 @@ -341,7 +341,7 @@ Mid and East Antrim,N09000008,Knockagh,N10000806,Burleigh Hill,N08000810 ,,,,Grange,N08000824 ,,,,Maine,N08000832 ,,,,Portglenone,N08000834 -Mid Ulster,N09000009,Carntogher,N10000901,Lower Glenshane,N08000926 +Mid Ulster District Council,N09000009,Carntogher,N10000901,Lower Glenshane,N08000926 ,,,,Maghera,N08000927 ,,,,Swatragh,N08000934 ,,,,Tamlaght O’Crilly,N08000935 @@ -381,7 +381,7 @@ Mid Ulster,N09000009,Carntogher,N10000901,Lower Glenshane,N08000926 ,,,,Caledon,N08000908 ,,,,Castlecaulfield,N08000909 ,,,,Fivemiletown,N08000920 -"Newry, Mourne and Down",N09000010,Slieve Gullion,N10001006,Bessbrook,N08001007 +"Newry, Mourne and Down District Council",N09000010,Slieve Gullion,N10001006,Bessbrook,N08001007 ,,,,Camlough,N08001010 ,,,,Crossmaglen,N08001014 ,,,,Forkhill,N08001023 @@ -422,7 +422,7 @@ Mid Ulster,N09000009,Carntogher,N10000901,Lower Glenshane,N08000926 ,,,,Derryboy,N08001016 ,,,,Kilmore,N08001026 ,,,,Saintfield,N08001036 -North Down and Ards,N09000011,Ards Peninsula,N10001101,Ballywalter,N08001106 +Ards and North Down Borough Council,N09000011,Ards Peninsula,N10001101,Ballywalter,N08001106 ,,,,Carrowdore,N08001110 ,,,,Kircubbin,N08001128 ,,,,Loughries,N08001129 From f4e27f308982e11a604804a57eed2b4eaecb47c1 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 18 Dec 2015 15:21:57 +0000 Subject: [PATCH 20/22] Make adding gss codes to ni areas work for real data We incorporate the feedback from mysociety about running the `mapit_UK_add_gss_codes_to_ni_areas` command against their real data. Because we're doing name matches we need to change our naive `names_name__iexact` match and sanitize the data a bit. --- .../mapit_UK_add_gss_codes_to_ni_areas.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py b/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py index b343a8b7..b8151354 100644 --- a/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py +++ b/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py @@ -81,19 +81,29 @@ def update_lgws_for_lge(self, wards, lgd, lge): ' LGD "%s" to add GSS "%s" to') % (ward_name, lge.name, lgd.name, ward_gss_code) def fetch_and_update_area(self, area_source, area_type, area_name, area_gss_code): + area_name = area_name.replace('St. ', 'St ') try: area = area_source.get( country=self.country, type__code=area_type, - names__name__iexact=area_name, + name__iexact=area_name, generation_low__lte=self.current_generation, generation_high__gte=self.current_generation ) + except Area.DoesNotExist: + try: + area = area_source.get( + country=self.country, type__code=area_type, + name__istartswith=area_name, + generation_low__lte=self.current_generation, + generation_high__gte=self.current_generation + ) + except Area.DoesNotExist: + return None, False + else: area.generation_high = self.new_generation area.save() _code, code_created = area.codes.get_or_create(type=self.gss_code_type, code=area_gss_code) return area, code_created - except Area.DoesNotExist: - return None, False def report_result(self, message, code_created): print message, From c159ba177873f067a4af2ee51fecfc49629ad2e9 Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Fri, 18 Dec 2015 15:59:49 +0000 Subject: [PATCH 21/22] Allow add_x_to_ni_areas scripts work on 1st import If there are no active generations we set the "current" generation to the "new" generation. Otherwise we try to find objects in the 0th generation and this won't work. --- .../management/commands/mapit_UK_add_gss_codes_to_ni_areas.py | 4 ++++ .../management/commands/mapit_UK_add_names_to_ni_areas.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py b/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py index b8151354..0a332c8e 100644 --- a/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py +++ b/mapit_gb/management/commands/mapit_UK_add_gss_codes_to_ni_areas.py @@ -16,6 +16,10 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): self.current_generation = Generation.objects.current() self.new_generation = Generation.objects.new() + if Generation.objects.filter(active=True).count() == 0: + # Let this work if you are running it on your 1st import before + # activation + self.current_generation = self.new_generation self.country = Country.objects.get(code='N') if not self.new_generation: raise Exception("No new generation to be used for import!") diff --git a/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py b/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py index f521e918..e86da433 100644 --- a/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py +++ b/mapit_gb/management/commands/mapit_UK_add_names_to_ni_areas.py @@ -16,6 +16,10 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): self.current_generation = Generation.objects.current() self.new_generation = Generation.objects.new() + if Generation.objects.filter(active=True).count() == 0: + # Let this work if you are running it on your 1st import before + # activation + self.current_generation = self.new_generation self.country = Country.objects.get(code='N') if not self.new_generation: raise Exception("No new generation to be used for import!") From 0730197335aae74d4ebda251af8f002da1959d3b Mon Sep 17 00:00:00 2001 From: Murray Steele Date: Tue, 5 Jan 2016 16:09:17 +0000 Subject: [PATCH 22/22] Convert geometry to application projection in NI shape imports We used to import all the NI shapes with an srid of 29902 (the Irish grid [1]). For some reason when the geometry was extracted from the DB it was in 27700 (the GB grid [2]) but had not undergone any transformation from 29902 to 27700. Consequently the NI shapes were in the wrong place (covering Liverpool, North Wales and some of the Irish Sea). It's not clear how this happened, but we can fix it by always transforming the NI shape data from whatever srid it is provided as into the 27700 srid used by the rest of the UK data. Note that we actually use the `settings.MAPIT_AREA_SRID` srid and not 27700 directly as in most cases of a UK instance of mapit this will be 27700, but in the off chance it's not we don't want things to break. [1]: http://spatialreference.org/ref/epsg/tm65-irish-grid/ [2]: http://spatialreference.org/ref/epsg/27700/ --- mapit_gb/management/commands/mapit_UK_import_osni.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mapit_gb/management/commands/mapit_UK_import_osni.py b/mapit_gb/management/commands/mapit_UK_import_osni.py index 2f621232..77e4fc6e 100644 --- a/mapit_gb/management/commands/mapit_UK_import_osni.py +++ b/mapit_gb/management/commands/mapit_UK_import_osni.py @@ -13,6 +13,7 @@ # from django.contrib.gis.utils import LayerMapping from django.contrib.gis.gdal import DataSource from django.utils import six +from django.conf import settings from mapit.models import Area, Name, Generation, Country, Type, CodeType, NameType from mapit.management.command_utils import save_polygons, fix_invalid_geos_geometry @@ -227,12 +228,14 @@ class AreaCodeShapefileInterpreter(object): def __init__(self, srid): self.srid = srid - # Transform all shapefile geometry to 29902 for consistency if it's - # not already in that projection + # Transform all shapefile geometry to the MAPIT_AREA_SRID if it's + # not already in that projection - otherwise the data is assumed to + # be in that projection when saved, regardless of the srid specified + # on the geometry object def transform_geom(self, geom): geom.srid = self.srid - if not(self.srid == 29902): - geom.transform(29902) + if not(self.srid == settings.MAPIT_AREA_SRID): + geom.transform(settings.MAPIT_AREA_SRID) return geom class WMC(AreaCodeShapefileInterpreter):