Merge pull request #265 from wxtim/tutorial.get_wind_obs_from_hem_hem

tutorial workflow - added alternative wind obs data retrieval
cylc · Sep 8, 2021 · ebb125b · ebb125b
2 parents bf9a145 + d1e8e8f
commit ebb125b
Show file tree

Hide file tree

Showing 9 changed files with 292 additions and 37 deletions.
diff --git a/cylc/doc/etc/tutorial/cylc-forecasting-workflow/bin/get-observations b/cylc/doc/etc/tutorial/cylc-forecasting-workflow/bin/get-observations
@@ -32,9 +32,12 @@ Environment Variables:
 
 """
 
+from pathlib import Path
 from datetime import datetime
 import json
 import os
+import re
+import requests
 import urllib3
 
 
@@ -64,6 +67,10 @@ ORDINATES = {
 }
 
 
+class NoDataException(Exception):
+    ...
+
+
 def get_datapoint_data(site_id, time, api_key):
     """Get weather data from the DataPoint service."""
     # The URL required to get the data.
@@ -107,17 +114,113 @@ def extract_observations(data):
             data['Period']['Rep']['S']]
 
 
+def reference_lat_long(site_id):
+    """Extract lat-long from a reference file."""
+    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
+    info = Path(src).read_text()
+    siteinfo = re.findall(f'^{int(site_id):05d}.*', info, re.MULTILINE)
+    if not siteinfo:
+        raise NoDataException(
+            f'Cannot find site with ID {site_id} in reference file {src}'
+        )
+    siteinfo = siteinfo[0].split()
+    lat = siteinfo[-4]
+    lon = siteinfo[-3]
+    return lat, lon
+
+
+def synop_grab(site_id, time):
+    """Get obs data from a resonably reliable 3rd Party source.
+
+    n.b. In theory the remote site has some limits on how many requests
+    you throw at it, but this is in the tens of thousands of lines, and
+    this request should get just 1.
+    """
+    # Construct a Request:
+    time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime('%Y%m%d%H%M')
+    url = (
+        f'https://www.ogimet.com/cgi-bin/getsynop?block={int(site_id):05d}'
+        f'&begin={time}&end={time}'
+    )
+    req = requests.get(url)
+    print(f'Requests Returned: {req.text}')
+
+    if req.status_code != 200:
+        # If request has failed return False.
+        raise NoDataException(f'Request for data Failed for {url}')
+
+    # See https://allaboutweather.tripod.com/synopcode.htm for an explanation
+    # of what this regex is looking for: Synopcode dd and ff
+    regex_wind = re.compile(
+        r'AAXX [\d\/]{5} [\d\/]{5} [\d\/]{5} [\d\/]'
+        r'(?P<direction>\d\d)(?P<speed>\d\d) 1'
+    )
+    try:
+        data = regex_wind.search(req.text).groupdict()
+    except AttributeError:
+        raise NoDataException(
+            f'Request for data failed, raw request was {req.text}')
+
+    # Parse the direction from 10's of degrees to degrees:
+    data['direction'] = str(int(data['direction']) * 10)
+
+    # Convert data in KT to MPH:
+    data['speed'] = str(int(data['speed']) * 1.15078)
+
+    # Get lat and long from MO Data:
+    lat, lon = reference_lat_long(site_id)
+
+    return [lat, lon, data['direction'], data['speed']]
+
+
+def get_nearby_site(site_id, badsites):
+    """Use Pythagoras to find the next nearest site."""
+    lat, lon = reference_lat_long(site_id)
+    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
+    info = Path(src).read_text()
+    dist = 100
+    result = 0
+    for site in info.split('\n')[6:]:
+        site = site.split()
+        slat = site[-4]
+        slon = site[-3]
+        x = float(slat) - float(lat)
+        y = float(slon) - float(lon)
+        ndist = (x ** 2 + y ** 2) ** 0.5
+        if ndist < dist and site[0] not in badsites:
+            dist = ndist
+            result = site
+    print('using {} (dist {})'.format(" ".join(result[1:][:-5]), dist))
+    return int(result[0]), dist
+
+
+
 def main(site_id, api_key=None):
     cycle_point = os.environ['CYLC_TASK_CYCLE_POINT']
 
-    if api_key:
-        print('Attempting to get weather data from DataPoint...')
-        data = get_datapoint_data(site_id, cycle_point, api_key)
-    else:
-        print('No API key provided, falling back to archived data')
-        data = get_archived_data(site_id, cycle_point)
+    # Try to get the information from SYNOPS.
+    dist = 0.0
+    badsites = []
+    fails = True
+    obs = None
+    while dist < 1 and fails:
+        try:
+            obs = synop_grab(site_id, cycle_point)
+            fails = False
+        except NoDataException:
+            badsites.append(f'{int(site_id):05d}')
+            site_id, dist = get_nearby_site(site_id, badsites)
+
+    if obs is None:
+        if api_key:
+            print('Attempting to get weather data from DataPoint...')
+            data = get_datapoint_data(site_id, cycle_point, api_key)
+        else:
+            print('No API key provided, falling back to archived data')
+            data = get_archived_data(site_id, cycle_point)
+
+        obs = extract_observations(data)
 
-    obs = extract_observations(data)
 
     # Write observations.
     with open('wind.csv', 'w+') as data_file:

diff --git a/cylc/doc/etc/tutorial/cylc-forecasting-workflow/etc/met-office-sites.dat b/cylc/doc/etc/tutorial/cylc-forecasting-workflow/etc/met-office-sites.dat
@@ -0,0 +1,152 @@
+# Retrieved from 
+# https://www.metoffice.gov.uk/binaries/content/assets/metofficegovuk/pdf/data/uk_synop_station_list_-_wmo_numbered_stations.pdf
+# on 2021/07/14
+# Observational Products - September 2019
+# UK SYNOP station list - WMO numbered stations
+WMO Name Alt (m) Latitude Longitude Country Date
+03002 BALTASOUND NO 2 15 60.74831 -0.85566 SCO 1983
+03005 LERWICK 82 60.13893 -1.18491 SCO 1922
+03008 FAIR ISLE 57 59.52607 -1.62959 SCO 1974
+03017 KIRKWALL 26 58.95345 -2.90149 SCO 1950
+03023 SOUTH UIST RANGE 4 57.35754 -7.38478 SCO 1996
+03026 STORNOWAY AIRPORT 15 58.21345 -6.31882 SCO 1873
+03031 LOCH GLASCARNOCH 269 57.7248 -4.89547 SCO 1992
+03034 AULTBEA NO 2 11 57.85886 -5.63277 SCO 1986
+03039 BEALACH NA BA NO 2 773 57.41786 -5.6886 SCO 2001
+03041 AONACH MOR 1130 56.8221 -4.9691 SCO 1992
+03044 ALTNAHARRA NO 2 81 58.28771 -4.44237 SCO 1983
+03047 TULLOCH BRIDGE 249 56.86656 -4.70797 SCO 1982
+03062 TAIN RANGE 4 57.81877 -3.96665 SCO 1989
+03063 AVIEMORE 228 57.20627 -3.82833 SCO 1982
+03065 CAIRNGORM SUMMIT 1237 57.11615 -3.64307 SCO 1984
+03066 KINLOSS 5 57.64547 -3.56347 SCO 1950
+03068 LOSSIEMOUTH 7 57.71126 -3.32335 SCO 1941
+03072 CAIRNWELL 928 56.87898 -3.42096 SCO 1993
+03075 WICK AIRPORT 36 58.4537 -3.08996 SCO 1873
+03080 ABOYNE NO 2 140 57.07576 -2.841 SCO 1989
+03088 INVERBERVIE NO 2 134 56.85189 -2.26594 SCO 1987
+03091 DYCE 65 57.20486 -2.20531 SCO 1942
+03092 PETERHEAD HARBOUR 15 57.50243 -1.77425 SCO 1979
+03100 TIREE 9 56.49982 -6.88057 SCO 1926
+03105 ISLAY: PORT ELLEN 17 55.68125 -6.24968 SCO 1968
+03111 MACHRIHANISH 10 55.44078 -5.69679 SCO 1965
+03132 WEST FREUGH 11 54.85907 -4.9353 SCO 1943
+03134 GLASGOW, BISHOPTON 59 55.90673 -4.53247 SCO 1998
+03136 PRESTWICK, GANNET 27 55.51531 -4.58466 SCO 1993
+03144 STRATHALLAN AIRFIELD 35 56.32634 -3.72871 SCO 1989
+03148 GLEN OGLE 564 56.42338 -4.32003 SCO 1996
+03153 DUNDRENNAN 113 54.80332 -4.0079 SCO 1983
+03155 DRUMALBIN 245 55.6271 -3.73615 SCO 1984
+03158 CHARTERHALL 112 55.70858 -2.38482 SCO 1987
+03162 ESKDALEMUIR 236 55.31185 -3.20685 SCO 1908
+03166 EDINBURGH, GOGARBANK 57 55.92832 -3.34434 SCO 1998
+03171 LEUCHARS 10 56.37734 -2.862 SCO 1921
+03204 RONALDSWAY 16 54.08521 -4.63187 IOM 1946
+03210 ST BEES HEAD NO 2 124 54.51775 -3.61476 ENG 1983
+03212 KESWICK 81 54.61396 -3.15662 ENG 1983
+03214 WALNEY ISLAND 15 54.12489 -3.25791 ENG 1971
+03220 CARLISLE 28 54.93428 -2.96364 ENG 1961
+03224 SPADEADAM NO 2 285 55.05018 -2.55444 ENG 1986
+03225 SHAP 252 54.50208 -2.6849 ENG 1982
+03226 WARCOP RANGE 227 54.57243 -2.41318 ENG 1983
+03227 GREAT DUN FELL NO 2 847 54.68401 -2.45132 ENG 1993
+03230 REDESDALE CAMP 211 55.28504 -2.27941 ENG 1979
+03238 ALBEMARLE 142 55.01979 -1.88167 ENG 2002
+03240 BOULMER 23 55.42085 -1.60126 ENG 1975
+03257 LEEMING 33 54.29698 -1.53301 ENG 1944
+03265 TOPCLIFFE 25 54.20471 -1.39013 ENG 1947
+03266 LINTON ON OUSE 14 54.04521 -1.25114 ENG 1958
+03275 LOFTUS 158 54.56284 -0.86353 ENG 1996
+03281 FYLINGDALES 262 54.35873 -0.67166 ENG 1961
+03292 BRIDLINGTON MRSC 15 54.09397 -0.17585 ENG 1987
+03301 MONA 60 53.26051 -4.37599 WAL 2009
+03302 VALLEY 10 53.25262 -4.53639 WAL 1930
+03305 CAPEL CURIG NO 3 216 53.09416 -3.94138 WAL 1993
+03313 RHYL NO 2 77 53.2593 -3.50882 WAL 1986
+03316 CROSBY 9 53.49724 -3.05764 ENG 1983
+03318 BLACKPOOL, SQUIRES GATE 10 53.77481 -3.03782 ENG 1942
+03321 HAWARDEN AIRPORT 11 53.17543 -2.98633 WAL 1944
+03330 LEEK, THORNCLIFFE 298 53.12784 -1.98139 ENG 2002
+03344 BINGLEY, NO 2 262 53.81122 -1.86676 ENG 1972
+03346 EMLEY MOOR NO 2 267 53.61206 -1.66849 ENG 2011
+03351 ROSTHERNE NO 2 35 53.36006 -2.38195 ENG 2012
+03354 NOTTINGHAM, WATNALL 117 53.00559 -1.25123 ENG 1941
+03373 SCAMPTON 57 53.30689 -0.54812 ENG 1963
+03377 WADDINGTON 68 53.1754 -0.52336 ENG 1946
+03379 CRANWELL 63 53.0312 -0.50357 ENG 1917
+03382 LECONFIELD 7 53.87459 -0.44176 ENG 1959
+03385 DONNA NOOK NO 2 8 53.47483 0.15287 ENG 1993
+03391 CONINGSBY 6 53.09384 -0.17286 ENG 1961
+03392 WAINFLEET NO 2 3 53.08835 0.27093 ENG 1988
+03405 ABERDARON 95 52.78887 -4.74137 WAL 1993
+03409 BALA 163 52.90757 -3.58429 WAL 1939
+03410 LAKE VYRNWY NO 2 360 52.75732 -3.46538 WAL 1988
+03414 SHAWBURY 72 52.79465 -2.66465 ENG 1944
+03462 WITTERING 74 52.61183 -0.46793 ENG 1955
+03469 HOLBEACH NO 2 3 52.87325 0.13851 ENG 1985
+03482 MARHAM 21 52.65134 0.56598 ENG 1951
+03488 WEYBOURNE 21 52.94936 1.12232 ENG 1986
+03502 ABERPORTH 133 52.13951 -4.5711 WAL 1941
+03507 SENNYBRIDGE NO 2 307 52.06329 -3.61463 WAL 1995
+03520 SHOBDON AIRFIELD 99 52.24297 -2.88582 ENG 1943
+03522 HEREFORD, CREDENHILL 76 52.08004 -2.80234 ENG 1999
+03529 PERSHORE 35 52.14843 -2.0412 ENG 1953
+03535 COLESHILL 96 52.48012 -1.69072 ENG 1997
+03544 CHURCH LAWFORD 107 52.35883 -1.33137 ENG 1983
+03559 CARDINGTON 29 52.10447 -0.4231 ENG 1952
+03560 BEDFORD 85 52.22696 -0.46536 ENG 1956
+03590 WATTISHAM 89 52.12387 0.95734 ENG 1959
+03604 MILFORD HAVEN CONSERVANCY BOARD 44 51.70883 -5.05334 WAL 1951
+03605 PEMBREY SANDS 3 51.71444 -4.36745 WAL 1993
+03629 AVONMOUTH 9 51.50597 -2.71663 ENG 1952
+03647 LITTLE RISSINGTON 210 51.8607 -1.69292 ENG 1942
+03649 BRIZE NORTON 82 51.75848 -1.57794 ENG 1968
+03658 BENSON 57 51.62038 -1.09862 ENG 1951
+03660 HIGH WYCOMBE, HQAIR 204 51.68181 -0.80682 ENG 1995
+03672 NORTHOLT 33 51.54865 -0.41692 ENG 1946
+03680 ROTHAMSTED 128 51.80671 -0.36017 ENG 1872
+03684 ANDREWSFIELD 87 51.89615 0.45056 ENG 1997
+03693 SHOEBURYNESS, LANDWICK 2 51.55475 0.827 ENG 1981
+03707 CHIVENOR 6 51.08915 -4.14856 ENG 1943
+03710 LISCOMBE 348 51.08687 -3.60894 ENG 1993
+03716 ST ATHAN 49 51.40544 -3.44074 WAL 1988
+03740 LYNEHAM 145 51.50323 -1.99233 ENG 1945
+03743 LARKHILL 132 51.20177 -1.80583 ENG 1920
+03746 BOSCOMBE DOWN 126 51.16179 -1.75458 ENG 1930
+03749 MIDDLE WALLOP 90 51.14987 -1.56994 ENG 1959
+03761 ODIHAM 118 51.23902 -0.94496 ENG 1963
+03768 SOUTH FARNBOROUGH 65 51.27996 -0.77259 ENG 1914
+03769 CHARLWOOD 67 51.14407 -0.22944 ENG 2002
+03770 LONDON, ST JAMES'S PARK 5 51.50476 -0.13109 ENG 1903
+03772 HEATHROW 25 51.47922 -0.45061 ENG 1947
+03781 KENLEY AIRFIELD 170 51.30399 -0.09154 ENG 1988
+03790 EAST MALLING 33 51.28756 0.4484 ENG 1925
+03796 LANGDON BAY 117 51.13361 1.34326 ENG 1984
+03797 MANSTON 49 51.34652 1.33539 ENG 1928
+03803 SCILLY: ST MARYS AIRPORT 31 49.91465 -6.29574 ENG 1979
+03808 CAMBORNE 87 50.21841 -5.32753 ENG 1978
+03809 CULDROSE 76 50.0844 -5.25707 ENG 1952
+03823 CARDINHAM, BODMIN 200 50.50216 -4.66685 ENG 1974
+03827 PLYMOUTH, MOUNTBATTEN 50 50.35493 -4.12097 ENG 1920
+03840 DUNKESWELL AERODROME 252 50.86023 -3.24013 ENG 1964
+03844 EXETER AIRPORT NO 2 27 50.73717 -3.40579 ENG 2009
+03853 YEOVILTON 20 51.00639 -2.64278 ENG 1964
+03857 ISLE OF PORTLAND 52 50.52215 -2.45562 ENG 1963
+03862 HURN 10 50.77946 -1.83622 ENG 1951
+03866 WIGHT: ST CATHERINES POINT 20 50.57566 -1.29693 ENG 1924
+03872 THORNEY ISLAND 4 50.81479 -0.92247 ENG 1941
+03876 SHOREHAM AIRPORT 2 50.83619 -0.2935 ENG 1983
+03882 HERSTMONCEUX, WEST END 52 50.89096 0.31655 ENG 1992
+03894 GUERNSEY: AIRPORT 101 49.43292 -2.59812 CHI 1949
+03895 JERSEY: AIRPORT 84 49.20965 -2.1943 CHI 1952
+03901 THOMASTOWN 72 54.32962 -7.59493 NI 2012
+03904 CASTLEDERG 49 54.70709 -7.57748 NI 1963
+03907 MAGILLIGAN NO 2 6 55.16039 -6.94924 NI 2011
+03911 LOUGH FEA 225 54.72113 -6.81467 NI 1965
+03915 PORTGLENONE 64 54.86534 -6.45781 NI 1989
+03916 BALLYPATRICK FOREST 156 55.18063 -6.15436 NI 1961
+03917 ALDERGROVE 63 54.66365 -6.22534 NI 1926
+03918 CASTOR BAY 15 54.50313 -6.34133 NI 2002
+03923 GLENANNE NO 2 161 54.23724 -6.5035 NI 1985
+03930 ALMONDSBURY 75 51.55138 -2.55933 ENG 2018
+03931 TIBENHAM AIRFIELD 52 52.45663 1.16219 ENG 2015
diff --git a/cylc/doc/etc/tutorial/cylc-forecasting-workflow/flow.cylc b/cylc/doc/etc/tutorial/cylc-forecasting-workflow/flow.cylc
@@ -2,7 +2,7 @@
     UTC mode = True
 [task parameters]
     # A list of the weather stations we will be fetching observations from.
-    station = camborne, heathrow, shetland, belmullet
+    station = camborne, heathrow, shetland, aldergrove
     # A list of the sites we will be generating forecasts for.
     site = exeter
 
@@ -50,9 +50,9 @@
             # To use archived data comment this line out.
             API_KEY = DATAPOINT_API_KEY
 
-    [[get_observations<station=belmullet>]]
+    [[get_observations<station=aldergrove>]]
         [[[environment]]]
-            SITE_ID = 3976
+            SITE_ID = 3917
     [[get_observations<station=camborne>]]
         [[[environment]]]
             SITE_ID = 3808

diff --git a/src/tutorial/runtime/configuration-consolidation/families.rst b/src/tutorial/runtime/configuration-consolidation/families.rst
@@ -99,10 +99,10 @@ Runtime
            inherit = GET_OBSERVATIONS
            [[[environment]]]
                SITE_ID = 3005
-       [[get_observations_belmullet]]
+       [[get_observations_aldergrove]]
            inherit = GET_OBSERVATIONS
            [[[environment]]]
-               SITE_ID = 3976
+               SITE_ID = 3917
 
 
 Graphing
@@ -128,7 +128,7 @@ Graphing
    get_observations_heathrow => consolidate_observations
    get_observations_camborne => consolidate_observations
    get_observations_shetland => consolidate_observations
-   get_observations_belmullet => consolidate_observations
+   get_observations_aldergrove => consolidate_observations
 
 .. ifnotslides::
 

diff --git a/src/tutorial/runtime/configuration-consolidation/index.rst b/src/tutorial/runtime/configuration-consolidation/index.rst
@@ -29,10 +29,10 @@ Consolidating Configuration
               [[[environment]]]
                   SITE_ID = 3005
                   API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-          [[get_observations_belmullet]]
+          [[get_observations_aldergrove]]
               script = get-observations
               [[[environment]]]
-                  SITE_ID = 3976
+                  SITE_ID = 3917
                   API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 
 .. ifnotslides::
@@ -63,7 +63,7 @@ Consolidating Configuration
       [scheduling]
           [[graph]]
               T00/PT3H = """
-                  get_observations_belmullet => consolidate_observations
+                  get_observations_aldergrove => consolidate_observations
                   get_observations_camborne => consolidate_observations
                   get_observations_heathrow => consolidate_observations
                   get_observations_shetland => consolidate_observations