diff --git a/Pipfile b/Pipfile index b610f9d6..4eca6e67 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ python-dateutil = "*" retry = "*" black = "*" rich = "*" +arcgis2geojson = "*" [dev-packages] pytest-mypy-plugins = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5fdafb83..81c6dc05 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "bd58710c5f6cd17cde12030bb1e584a2e26f9269ffe455be920c54845a2eac0c" + "sha256": "d4e6c72783a0dcb82e0a76ef5a7bd6474f108b57d654ea07d1746e960524eb2c" }, "pipfile-spec": 6, "requires": { @@ -23,6 +23,14 @@ ], "version": "==1.4.4" }, + "arcgis2geojson": { + "hashes": [ + "sha256:07336ba30b783099dd2bfb4d521a825736b69966a580d2f92413afe0bdc4a41a", + "sha256:dc35be2c52184e720ef3b5df96fa9ba1624f16feb25c04a1521e5d5567a5e992" + ], + "index": "pypi", + "version": "==2.0.1" + }, "astroid": { "hashes": [ "sha256:4db03ab5fc3340cf619dbc25e42c2cc3755154ce6009469766d7143d1fc2ee4e", @@ -474,7 +482,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytest": { @@ -615,7 +623,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "soupsieve": { @@ -638,7 +646,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "traitlets": { @@ -712,31 +720,32 @@ }, "mypy": { "hashes": [ - "sha256:0d0a87c0e7e3a9becdfbe936c981d32e5ee0ccda3e0f07e1ef2c3d1a817cf73e", - "sha256:25adde9b862f8f9aac9d2d11971f226bd4c8fbaa89fb76bdadb267ef22d10064", - "sha256:28fb5479c494b1bab244620685e2eb3c3f988d71fd5d64cc753195e8ed53df7c", - "sha256:2f9b3407c58347a452fc0736861593e105139b905cca7d097e413453a1d650b4", - "sha256:33f159443db0829d16f0a8d83d94df3109bb6dd801975fe86bacb9bf71628e97", - "sha256:3f2aca7f68580dc2508289c729bd49ee929a436208d2b2b6aab15745a70a57df", - "sha256:499c798053cdebcaa916eef8cd733e5584b5909f789de856b482cd7d069bdad8", - "sha256:4eec37370483331d13514c3f55f446fc5248d6373e7029a29ecb7b7494851e7a", - "sha256:552a815579aa1e995f39fd05dde6cd378e191b063f031f2acfe73ce9fb7f9e56", - "sha256:5873888fff1c7cf5b71efbe80e0e73153fe9212fafdf8e44adfe4c20ec9f82d7", - "sha256:61a3d5b97955422964be6b3baf05ff2ce7f26f52c85dd88db11d5e03e146a3a6", - "sha256:674e822aa665b9fd75130c6c5f5ed9564a38c6cea6a6432ce47eafb68ee578c5", - "sha256:7ce3175801d0ae5fdfa79b4f0cfed08807af4d075b402b7e294e6aa72af9aa2a", - "sha256:9743c91088d396c1a5a3c9978354b61b0382b4e3c440ce83cf77994a43e8c521", - "sha256:9f94aac67a2045ec719ffe6111df543bac7874cee01f41928f6969756e030564", - "sha256:a26f8ec704e5a7423c8824d425086705e381b4f1dfdef6e3a1edab7ba174ec49", - "sha256:abf7e0c3cf117c44d9285cc6128856106183938c68fd4944763003decdcfeb66", - "sha256:b09669bcda124e83708f34a94606e01b614fa71931d356c1f1a5297ba11f110a", - "sha256:cd07039aa5df222037005b08fbbfd69b3ab0b0bd7a07d7906de75ae52c4e3119", - "sha256:d23e0ea196702d918b60c8288561e722bf437d82cb7ef2edcd98cfa38905d506", - "sha256:d65cc1df038ef55a99e617431f0553cd77763869eebdf9042403e16089fe746c", - "sha256:d7da2e1d5f558c37d6e8c1246f1aec1e7349e4913d8fb3cb289a35de573fe2eb" + "sha256:053b92ebae901fc7954677949049f70133f2f63e3e83dc100225c26d6a46fe95", + "sha256:08cf1f31029612e1008a9432337ca4b1fbac989ff7c8200e2c9ec42705cd4c7b", + "sha256:18753a8bb9bcf031ff10009852bd48d781798ecbccf45be5449892e6af4e3f9f", + "sha256:1cd241966a35036f936d4739bd71a1c64e15f02bf7d12bb2815cccfb2993a9de", + "sha256:307a6c047596d768c3d689734307e47a91596eb9dbb67cfdf7d1fd9117b27f13", + "sha256:4a622faa3be76114cdce009f8ec173401494cf9e8f22713e7ae75fee9d906ab3", + "sha256:4b54518e399c3f4dc53380d4252c83276b2e60623cfc5274076eb8aae57572ac", + "sha256:5ddd8f4096d5fc2e7d7bb3924ac22758862163ad2c1cdc902c4b85568160e90a", + "sha256:61b10ba18a01d05fc46adbf4f18b0e92178f6b5fd0f45926ffc2a408b5419728", + "sha256:7845ad3a31407bfbd64c76d032c16ab546d282930f747023bf07c17b054bebc5", + "sha256:79beb6741df15395908ecc706b3a593a98804c1d5b5b6bd0c5b03b67c7ac03a0", + "sha256:8183561bfd950e93eeab8379ae5ec65873c856f5b58498d23aa8691f74c86030", + "sha256:91211acf1485a1db0b1261bc5f9ed450cba3c0dfd8da0a6680e94827591e34d7", + "sha256:97be0e8ed116f7f79472a49cf06dd45dd806771142401f684d4f13ee652a63c0", + "sha256:9941b685807b60c58020bb67b3217c9df47820dcd00425f55cdf71f31d3c42d9", + "sha256:a85c6759dcc6a9884131fa06a037bd34352aa3947e7f5d9d5a35652cc3a44bcd", + "sha256:bc61153eb4df769538bb4a6e1045f59c2e6119339690ec719feeacbfc3809e89", + "sha256:bf347c327c48d963bdef5bf365215d3e98b5fddbe5069fc796cec330e8235a20", + "sha256:c86e3f015bfe7958646825d41c0691c6e5a5cd4015e3409b5c29c18a3c712534", + "sha256:c8bc628961cca4335ac7d1f2ed59b7125d9252fe4c78c3d66d30b50162359c99", + "sha256:da914faaa80c25f463913da6db12adba703822a768f452f29f75b40bb4357139", + "sha256:e8577d30daf1b7b6582020f539f76e78ee1ed64a0323b28c8e0333c45db9369f", + "sha256:f208cc967e566698c4e30a1f65843fc88d8da05a8693bac8b975417e0aee9ced" ], "markers": "python_version >= '3.5'", - "version": "==0.812" + "version": "==0.901" }, "mypy-extensions": { "hashes": [ @@ -774,7 +783,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pystache": { @@ -839,44 +848,9 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, - "typed-ast": { - "hashes": [ - "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace", - "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff", - "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266", - "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528", - "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6", - "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808", - "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4", - "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363", - "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341", - "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04", - "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41", - "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e", - "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3", - "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899", - "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805", - "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c", - "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c", - "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39", - "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a", - "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3", - "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7", - "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f", - "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075", - "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0", - "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40", - "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428", - "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927", - "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3", - "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f", - "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65" - ], - "version": "==1.4.3" - }, "typing-extensions": { "hashes": [ "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", diff --git a/lgsf/conf/__init__.py b/lgsf/conf/__init__.py index 2c711e64..98a151a3 100644 --- a/lgsf/conf/__init__.py +++ b/lgsf/conf/__init__.py @@ -23,6 +23,7 @@ def __init__(self): "councillors", "templates", "metadata", + "polling_stations" # 'parties', # "scrapers", # 'reconcilers', diff --git a/lgsf/polling_stations/__init__.py b/lgsf/polling_stations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lgsf/polling_stations/commands.py b/lgsf/polling_stations/commands.py new file mode 100644 index 00000000..c41944ed --- /dev/null +++ b/lgsf/polling_stations/commands.py @@ -0,0 +1,101 @@ +from rich.progress import Progress + +from lgsf.commands.base import PerCouncilCommandBase +from lgsf.path_utils import load_scraper, load_council_info + + + +class Command(PerCouncilCommandBase): + command_name = "polling_stations" + + def add_arguments(self, parser): + parser.add_argument( + "--check-only", + action="store_true", + help="Just check for updated pages, don't scrape anything", + ) + parser.add_argument( + "--list-missing", + action="store_true", + help="Print missing councils", + ) + parser.add_argument( + "--list-disabled", + action="store_true", + help="Print disabled councils", + ) + + def _run_single(self, scraper, progress, summary): + try: + progress.console.print( + scraper.options["council"] + ) + scraper.run() + summary["completed"] += 1 + except KeyboardInterrupt: + raise + except: + if self.options.get("verbose"): + raise + summary["failed"] += 1 + progress.console.print( + "Error running asdasd {}, see {} for more".format( + self.options["council"], scraper._error_file_name() + ), + style="red", + ) + + + def handle(self, options): + self.options = options + if options["list_missing"]: + self.output_missing() + + if options["list_disabled"]: + self.output_disabled() + + self.output_status() + + self.normalise_codes() + to_run = self.councils_to_run() + summary = { + "completed": 0, + "missing scraper": 0, + "failed": 0, + "skipped": 0, + } + with Progress() as progress: + tasks = { + "total": progress.add_task(description=f"Total", total=len(to_run)), + } + + while not progress.finished: + for council in to_run: + self.options["council"] = council + self.options["council_info"] = load_council_info(council) + scraper_cls = load_scraper(council, self.command_name) + if not scraper_cls: + summary["missing scraper"] += 1 + continue + with scraper_cls((self.options), progress.console) as scraper: + should_run = True + if scraper.disabled: + should_run = False + + if should_run and options["refresh"]: + if scraper.run_since(): + should_run = False + + if should_run and options["tags"]: + required_tags = set(options["tags"].split(",")) + scraper_tags = set(scraper.get_tags) + if not required_tags.issubset(scraper_tags): + should_run = False + + if should_run: + self._run_single(scraper, progress, summary) + else: + summary["skipped"] += 1 + + progress.update(tasks["total"], advance=1) + self.console.print(summary) \ No newline at end of file diff --git a/lgsf/polling_stations/models.py b/lgsf/polling_stations/models.py new file mode 100644 index 00000000..2dc09866 --- /dev/null +++ b/lgsf/polling_stations/models.py @@ -0,0 +1,26 @@ +import json + +class PollingStationsList: + def __init__( + self, stations + ): + self.stations = stations + + def as_file_name(self): + return "stations" + + def as_json(self): + return json.dumps(self.stations, indent=4) + + +class PollingDistrictsList: + def __init__( + self, districts + ): + self.districts = districts + + def as_file_name(self): + return "districts" + + def as_json(self): + return json.dumps(self.districts, indent=4) diff --git a/lgsf/polling_stations/scrapers/__init__.py b/lgsf/polling_stations/scrapers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lgsf/polling_stations/scrapers/arcgis_scraper.py b/lgsf/polling_stations/scrapers/arcgis_scraper.py new file mode 100644 index 00000000..ebe5e3cf --- /dev/null +++ b/lgsf/polling_stations/scrapers/arcgis_scraper.py @@ -0,0 +1,57 @@ +import json +from arcgis2geojson import arcgis2geojson + +from lgsf.polling_stations.models import PollingStationsList, PollingDistrictsList +from lgsf.scrapers.base import ScraperBase +from lgsf.polling_stations.scrapers.common import ( + # BaseScraper, + # get_data_from_url, + # save, + summarise, + sync_db_to_github, + truncate, PollingStationScraperBase, +) + +class ArcGisScraper(PollingStationScraperBase): + + def make_geometry(self, feature): + return json.dumps(arcgis2geojson(feature), sort_keys=True) + + def get_data(self, url): # pragma: no cover + response = self.get(url) + data_str = response.content + data = json.loads(data_str.decode(self.encoding)) + return (data_str, data) + + + def process_feature(self, feature, fields=None): + # assemble record + record = { + 'council_id': self.council_id, + 'geometry': self.make_geometry(feature), + } + for field in fields: + value = feature['attributes'][field['name']] + if isinstance(value, str): + record[field['name']] = value.strip() + else: + record[field['name']] = value + return record + + + def scrape(self, url, type='features'): + # load json + data_str, data = self.get_data(url) + print(f"found {len(data['features'])} {type}") + + # grab field names + fields = data['fields'] + features = data['features'] + + return self.process_features(features, fields) + + + # print summary + # summarise(self.table) + + # self.store_history(data_str, self.council_id) diff --git a/lgsf/polling_stations/scrapers/common.py b/lgsf/polling_stations/scrapers/common.py new file mode 100644 index 00000000..e8012255 --- /dev/null +++ b/lgsf/polling_stations/scrapers/common.py @@ -0,0 +1,113 @@ +import abc +import datetime +import json +import os +# import scraperwiki +import urllib.request +from collections import OrderedDict +# from commitment import GitHubCredentials, GitHubClient +from hashlib import sha1 +from retry import retry +from urllib.error import HTTPError + +from lgsf.polling_stations.models import PollingStationsList, PollingDistrictsList +from lgsf.scrapers.base import ScraperBase + + + +def truncate(table): + pass + +def summarise(table): + pass + +def get_github_credentials(): + return GitHubCredentials( + repo=os.environ['MORPH_GITHUB_POLLING_REPO'], + name=os.environ['MORPH_GITHUB_USERNAME'], + email=os.environ['MORPH_GITHUB_EMAIL'], + api_key=os.environ['MORPH_GITHUB_API_KEY'] + ) + +def format_json(json_str, exclude_keys=None): + data = json.loads(json_str, object_pairs_hook=OrderedDict) + if isinstance(data, dict) and exclude_keys: + for key in exclude_keys: + data.pop(key, None) + return json.dumps(data, sort_keys=True, indent=4 ) + + +def sync_file_to_github(council_id, file_name, content): + try: + creds = get_github_credentials() + g = GitHubClient(creds) + path = "%s/%s" % (council_id, file_name) + + # if we haven't specified an extension, assume .json + if '.' not in path: + path = "%s.json" % (path) + + g.push_file(content, path, 'Update %s at %s' %\ + (path, str(datetime.datetime.now()))) + except KeyError: + # if no credentials are defined in env vars + # just ignore this step + pass + + +def sync_db_to_github(council_id, table_name, key): + # content = dump_table_to_json(table_name, key) + # sync_file_to_github(council_id, table_name, content) + pass + + + + + +class PollingStationScraperBase(ScraperBase, metaclass=abc.ABCMeta): + + store_raw_data = False + + def __init__(self, options, console): + super().__init__(options, console) + self.council_id = self.options["council"] + self.stations = [] + self.districts = [] + + def save_stations(self): + station_list = PollingStationsList(self.stations) + # sync_db_to_github(self.council_id, self.table, self.key) + self.save_json(station_list) + + def save_districts(self): + districts_list = PollingDistrictsList(self.districts) + # sync_db_to_github(self.council_id, self.table, self.key) + self.save_json(districts_list) + + def run(self): + if self.stations_url: + self.stations = self.scrape(self.stations_url, "stations") + self.save_stations() + if self.districts_url: + self.districts = self.scrape(self.districts_url, "districts")# + self.save_districts() + + def process_features(self, features, fields=None): + feature_list = [] + for feature in features: + record = self.process_feature(feature, fields) + feature_list.append(record) + return feature_list + + @abc.abstractmethod + def scrape(self, url, type='features'): + pass + + @abc.abstractmethod + def get_data(self, url): + pass + + @abc.abstractmethod + def process_feature(self, feature, fields=None): + pass + diff --git a/lgsf/polling_stations/scrapers/xml_scrapers.py b/lgsf/polling_stations/scrapers/xml_scrapers.py new file mode 100644 index 00000000..20ecb8f4 --- /dev/null +++ b/lgsf/polling_stations/scrapers/xml_scrapers.py @@ -0,0 +1,60 @@ +import abc +from xml import etree + +from lgsf.polling_stations.scrapers.common import PollingStationScraperBase + + +class XmlScraper(PollingStationScraperBase, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def make_geometry(self, xmltree, element): + pass + + @property + @abc.abstractmethod + def feature_tag(self): + pass + + def get_data(self,url): # pragma: no cover + response = self.get(url) + return response + + def process_feature(self, feature, tree): + record = { + 'council_id': self.council_id, + 'geometry': self.make_geometry(tree, feature), + } + + # extract attributes and assemble record + for attribute in feature[0]: + if attribute.tag in self.fields: + if isinstance(attribute.text, str): + record[self.fields[attribute.tag]] = attribute.text.strip() + else: + record[self.fields[attribute.tag]] = attribute.text + return record + + def scrape(self): + + if not isinstance(self.pk, list): + self.pk = [self.pk] + + # load xml + data_str = self.get_data() + tree = etree.fromstring(data_str) + features = tree.findall(self.feature_tag) + print("found %i %s" % (len(features), self.table)) + + return self.process_features(features, fields) + + # print summary + summarise(self.table) + + self.store_history(data_str, self.council_id) + + def dump_fields(self): # pragma: no cover + data_str = self.get_data() + tree = etree.fromstring(data_str) + features = tree.findall(self.feature_tag) + for attribute in features[0][0]: + print(attribute.tag) \ No newline at end of file diff --git a/scrapers/EPS-epsom-and-ewell/polling_stations.py b/scrapers/EPS-epsom-and-ewell/polling_stations.py new file mode 100644 index 00000000..9b0737cb --- /dev/null +++ b/scrapers/EPS-epsom-and-ewell/polling_stations.py @@ -0,0 +1,24 @@ +from lgsf.polling_stations.scrapers.arcgis_scraper import ArcGisScraper + + +class Scraper(ArcGisScraper): + stations_url = "https://maps.epsom-ewell.gov.uk/getOWS.ashx?MapSource=EEBC/inspire&service=WFS&version=1.1.0&request=GetFeature&Typename=pollingstations" + stations_fields = { + "{http://mapserver.gis.umn.edu/mapserver}msGeometry": "msGeometry", + "{http://mapserver.gis.umn.edu/mapserver}psnumber": "psnumber", + "{http://mapserver.gis.umn.edu/mapserver}district": "district", + "{http://mapserver.gis.umn.edu/mapserver}address": "address", + "{http://mapserver.gis.umn.edu/mapserver}ward": "ward", + } + + districts_url = "https://maps.epsom-ewell.gov.uk/getOWS.ashx?MapSource=EEBC/inspire&service=WFS&version=1.1.0&request=GetFeature&Typename=pollingdistricts" + districts_fields = { + "{http://www.opengis.net/gml}boundedBy": "boundedBy", + "{http://mapserver.gis.umn.edu/mapserver}msGeometry": "msGeometry", + "{http://mapserver.gis.umn.edu/mapserver}district": "district", + "{http://mapserver.gis.umn.edu/mapserver}pollingplace": "pollingplace", + } + encoding = 'utf-8' + + station_key = "psnumber" + districts_key = "id" diff --git a/scrapers/WLV-wolverhampton/polling_stations.py b/scrapers/WLV-wolverhampton/polling_stations.py new file mode 100644 index 00000000..3d261de3 --- /dev/null +++ b/scrapers/WLV-wolverhampton/polling_stations.py @@ -0,0 +1,11 @@ +from lgsf.polling_stations.scrapers.arcgis_scraper import ArcGisScraper + + +class Scraper(ArcGisScraper): + stations_url = "https://maps.wolverhampton.gov.uk/arcgis/rest/services/PollingDistrictsStations/MapServer/0/query?where=OBJECTID+LIKE+%27%25%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryPolygon&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=OBJECTID&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&resultOffset=&resultRecordCount=&f=pjson" + districts_url = "https://maps.wolverhampton.gov.uk/arcgis/rest/services/PollingDistrictsStations/MapServer/1/query?where=OBJECTID+LIKE+%27%25%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryPoint&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=OBJECTID&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&resultOffset=&resultRecordCount=&f=pjson" + encoding = 'utf-8' + # table = 'stations' + key = 'OBJECTID' + +