feat: update to ERA Ontology. Version 3.0.1

bergmannjg · Sep 10, 2024 · 5f60b8a · 5f60b8a
1 parent bbcccff
commit 5f60b8a
Show file tree

Hide file tree

Showing 10 changed files with 223 additions and 98 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -3,13 +3,13 @@
   "isRoot": true,
   "tools": {
     "fable": {
-      "version": "4.12.2",
+      "version": "4.20.0",
       "commands": [
         "fable"
       ]
     },
     "fantomas": {
-      "version": "6.1.2",
+      "version": "6.3.12",
       "commands": [
         "fantomas"
       ]

diff --git a/erakg-datasparql-countries.json b/erakg-datasparql-countries.json
@@ -0,0 +1,30 @@
+
+{ "head": { "link": [], "vars": ["country"] },
+  "results": { "distinct": false, "ordered": true, "bindings": [
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/CZE" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/HRV" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/GBR" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/DEU" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/AUT" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/BEL" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/BGR" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/ESP" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/HUN" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/ITA" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/NLD" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/SVK" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/CHE" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/DNK" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/GRC" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/EST" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/FIN" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/FRA" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/LUX" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/LTU" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/LVA" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/POL" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/PRT" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/SVN" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/ROU" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/SWE" }},
+    { "country": { "type": "uri", "value": "http://publications.europa.eu/resource/authority/country/NOR" }} ] } }
diff --git a/erakg-datasparql-tracks.json b/erakg-datasparql-tracks.json
diff --git a/src/EraKGApi/Api.fs b/src/EraKGApi/Api.fs
@@ -1,6 +1,6 @@
 // see ERA knowledge graph  https://zenodo.org/record/6516745/files/[email protected]
 // see https://data-interop.era.europa.eu/endpoint
-// see sparql endpoint https://virtuoso.ecdp.tech.ec.europa.eu/sparql/
+// see sparql endpoint https://era.linkeddata.es/sparql/
 namespace EraKG
 
 open FSharp.Collections
@@ -71,7 +71,7 @@ module Api =
     let propTenClassification = "http://data.europa.eu/949/tenClassification"
     let propNetElements = "http://data.europa.eu/949/topology/netElements/"
 
-    let private endpoint = "https://virtuoso.ecdp.tech.ec.europa.eu/sparql"
+    let private endpoint = "https://era.linkeddata.es/sparql"
 
     let countrySplitChars = [| ';' |]
 
@@ -89,17 +89,19 @@ WHERE {{
 }}
 """
 
-    let loadCountriesData () : Async<string> =
-        Request.GetAsync endpoint (countriesQuery ()) Request.applicationSparqlResults
-
+    let loadCountriesData () : Async<QueryResults> =
+        async {
+            let! data = Request.GetAsync endpoint (countriesQuery ()) Request.applicationSparqlResults
+            return JsonSerializer.Deserialize(data)
+        }
 
     let private toCountryQuery (item: string) (countryArg: string) : string =
         countryArg.Split countrySplitChars
         |> Array.map (fun country ->
             $"{{ {item} era:inCountry <http://publications.europa.eu/resource/authority/country/{country}> . }}")
         |> String.concat " UNION "
 
-    let private operationalPointQuery (country: string) =
+    let private operationalPointQuery (country: string) (limit: int) (offset: int) =
         $"""
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX era: <http://data.europa.eu/949/>
@@ -116,13 +118,18 @@ WHERE {{
   ?operationalPoint era:opType ?opType .
   ?operationalPoint era:inCountry ?country .
   {toCountryQuery "?operationalPoint" country}
-}}
+}} LIMIT {limit} OFFSET {offset}
 """
 
-    let loadOperationalPointData (country: string) : Async<string> =
-        Request.GetAsync endpoint (operationalPointQuery country) Request.applicationSparqlResults
+    let loadOperationalPointData (country: string) (limit: int) (offset: int) : Async<QueryResults> =
+        async {
+            let! data =
+                Request.GetAsync endpoint (operationalPointQuery country limit offset) Request.applicationSparqlResults
+
+            return JsonSerializer.Deserialize(data)
+        }
 
-    let private sectionOfLineQuery (country: string) =
+    let private sectionOfLineQuery (country: string) (limit: int) (offset: int) =
         $"""
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX era: <http://data.europa.eu/949/>
@@ -140,11 +147,16 @@ WHERE {{
   ?sectionOfLine era:track ?track .
   ?sectionOfLine era:inCountry ?country .
   {toCountryQuery "?sectionOfLine" country}
-}}
+}} LIMIT {limit} OFFSET {offset}
 """
 
-    let loadSectionOfLineData (country: string) : Async<string> =
-        Request.GetAsync endpoint (sectionOfLineQuery country) Request.applicationSparqlResults
+    let loadSectionOfLineData (country: string) (limit: int) (offset: int) : Async<QueryResults> =
+        async {
+            let! data =
+                Request.GetAsync endpoint (sectionOfLineQuery country limit offset) Request.applicationSparqlResults
+
+            return JsonSerializer.Deserialize(data)
+        }
 
     let private trackOfLineQuery (lineName: string) (country: string) =
         $"""
@@ -200,16 +212,7 @@ SELECT ?p ?o WHERE {{
                 return "{\"items\":[]}"
         }
 
-    let private trackQuery (country: string) (n: int) =
-        let filter = if n < 9 then n.ToString() else (n.ToString() + "a-z")
-
-        let toFilter () : string =
-            linesWithError
-            |> Array.map (fun (line, _) -> $" FILTER(?line != \"{line}\") . ")
-            |> String.concat ""
-
-        let pattern = $"^[{filter}]"
-
+    let private trackQuery (country: string) (limit: int) (offset: int) =
         $"""
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX era: <http://data.europa.eu/949/>
@@ -221,20 +224,18 @@ WHERE {{
   
   ?sectionOfLine era:lineNationalId ?lineNationalId .
   ?lineNationalId rdfs:label ?line .
-  {toFilter ()}
-  FILTER(regex(?line, "{pattern}", "i")) .
   {toCountryQuery "?sectionOfLine" country}
-}}
+}} LIMIT {limit} OFFSET {offset}
 """
 
-    let loadTrackData (country: string) (n: int) : Async<string> =
+    let loadTrackData (country: string) (limit: int) (offset: int) : Async<Microdata> =
         async {
             try
-                let! data = Request.GetAsync endpoint (trackQuery country n) Request.applicationMicrodata
-                return data
+                let! data = Request.GetAsync endpoint (trackQuery country limit offset) Request.applicationMicrodata
+                return JsonSerializer.Deserialize<Microdata>(data)
             with e ->
                 fprintfn stderr "error: loadTrackData %s" e.Message
-                return "{\"items\":[]}"
+                return Microdata.empty
         }
 
     let private nationalRailwayLineQuery (country: string) =
@@ -255,8 +256,11 @@ WHERE {{
 }}
 """
 
-    let loadNationalRailwayLineData (country: string) : Async<string> =
-        Request.GetAsync endpoint (nationalRailwayLineQuery country) Request.applicationSparqlResults
+    let loadNationalRailwayLineData (country: string) : Async<QueryResults> =
+        async {
+            let! data = Request.GetAsync endpoint (nationalRailwayLineQuery country) Request.applicationSparqlResults
+            return JsonSerializer.Deserialize(data)
+        }
 
     let private tunnelQuery (country: string) =
         $"""
@@ -277,8 +281,11 @@ WHERE {{
 }}
 """
 
-    let loadTunnelData (country: string) : Async<string> =
-        Request.GetAsync endpoint (tunnelQuery country) Request.applicationSparqlResults
+    let loadTunnelData (country: string) : Async<QueryResults> =
+        async {
+            let! data = Request.GetAsync endpoint (tunnelQuery country) Request.applicationSparqlResults
+            return JsonSerializer.Deserialize(data)
+        }
 
     let private uriTypeToString (r: Rdf) (prefix: string) : string =
         if r.``type`` = "uri" then
@@ -296,7 +303,11 @@ WHERE {{
         uriTypeToString r "http://data.europa.eu/949/functionalInfrastructure/sectionsOfLine/"
 
     let private toNationalLines (r: Rdf) : string =
-        let splits = (uriTypeToString r "http://data.europa.eu/949/functionalInfrastructure/nationalLines/").Split [| '/' |]
+        let splits =
+            (uriTypeToString r "http://data.europa.eu/949/functionalInfrastructure/nationalLines/")
+                .Split
+                [| '/' |]
+
         Array.last splits // ignore country
 
     let private toUOPID (r: Rdf) : string =
@@ -309,22 +320,24 @@ WHERE {{
         (float splits.[0], float splits.[1])
 
     let private toRailwayLocation (r: Rdf) : RailwayLocation =
-        let line = uriTypeToString r "http://data.europa.eu/949/functionalInfrastructure/lineReferences/"
+        let line =
+            uriTypeToString r "http://data.europa.eu/949/functionalInfrastructure/lineReferences/"
+
         let index = line.LastIndexOf '_'
 
         if index > 0 then
-            let km = line.Substring(index+1)
+            let km = line.Substring(index + 1)
+
             { NationalIdentNum = line.Substring(0, index)
-              Kilometer = 
-                try float km
-                with e -> 
+              Kilometer =
+                try
+                    float km
+                with e ->
                     fprintfn stderr "error: '%s' %s" km e.Message
-                    0.0 
-            }
-        else 
+                    0.0 }
+        else
             { NationalIdentNum = line
-              Kilometer = 0.0 
-            }
+              Kilometer = 0.0 }
 
     let private toFloat (r: Rdf) : float =
         try
@@ -404,6 +417,11 @@ WHERE {{
         |> Map.values
         |> Seq.toArray
 
+    // may conatin extra data until '_'
+    let stripContactLineSystem (s: string) : string =
+        let index = s.LastIndexOf '_'
+        if (index <> -1) then s.Substring(index + 1) else s
+
     let toTrack (id: string) (tracks: Microdata) : Track =
         match tracks.items |> Array.tryFind (fun item -> item.id = id) with
         | Some item ->
@@ -423,7 +441,10 @@ WHERE {{
                 |> Option.map (fun s -> s.Substring(prefixLoadCapabilities.Length))
               contactLineSystem =
                 getValue propContactLineSystem
-                |> Option.map (fun s -> (System.Web.HttpUtility.UrlDecode(s)).Substring(prefixContactLineSystem.Length)) }
+                |> Option.map (fun s ->
+                    stripContactLineSystem (
+                        (System.Web.HttpUtility.UrlDecode(s)).Substring(prefixContactLineSystem.Length)
+                    )) }
         | None ->
             { id = System.Web.HttpUtility.UrlDecode(id.Substring(prefixTrack.Length))
               label = "directional track"
@@ -506,7 +527,8 @@ WHERE {{
 
             let queryResults: QueryResults = JsonSerializer.Deserialize res
 
-            let maximumPermittedSpeed = getValue queryResults propMaximumPermittedSpeed |> toIntValue
+            let maximumPermittedSpeed =
+                getValue queryResults propMaximumPermittedSpeed |> toIntValue
 
             let contactLineSystem =
                 getValue queryResults propContactLineSystem

diff --git a/src/EraKGApi/Request.fs b/src/EraKGApi/Request.fs
@@ -5,7 +5,7 @@ module Request =
     open System.Net
 
     let applicationQleverResults = "application/qlever-results+json"
-    
+
     let applicationSparqlResults = "application/sparql-results+json"
 
     let applicationMicrodata = "application/microdata+json"

diff --git a/src/EraKGApi/Sparql.fs b/src/EraKGApi/Sparql.fs
@@ -17,8 +17,34 @@ type Results = { bindings: Map<string, Rdf>[] }
 
 type QueryResults = { head: Head; results: Results }
 
+module QueryResults =
+    let concat (q1: QueryResults) (q2: QueryResults) =
+        { q1 with
+            head = if q1.head.vars.Length > 0 then q1.head else q2.head
+            results = { bindings = Array.concat [ q1.results.bindings; q2.results.bindings ] } }
+
+    let empty =
+        { head = { vars = Array.empty }
+          results = { bindings = Array.empty } }
+
+    let length (data: QueryResults) : int = data.results.bindings.Length
+
+    let fold (data: QueryResults[]) : QueryResults =
+        Array.fold (fun acc (r: QueryResults) -> concat acc r) empty data
+
 type Item =
     { id: string
       properties: Map<string, obj[]> }
 
 type Microdata = { items: Item[] }
+
+module Microdata =
+    let concat (q1: Microdata) (q2: Microdata) (chooser: (Item -> option<Item>)) : Microdata =
+        { items = Array.concat [ q1.items; q2.items |> Array.choose chooser ] }
+
+    let empty: Microdata = { items = Array.empty }
+
+    let length (data: Microdata) : int = data.items.Length
+
+    let fold (chooser: (Item -> option<Item>)) (data: Microdata[]) : Microdata =
+        Array.fold (fun acc (r: Microdata) -> concat acc r chooser) empty data
diff --git a/src/EraKGLoader/OsmRoutes.fs b/src/EraKGLoader/OsmRoutes.fs
@@ -11,19 +11,22 @@ module Api =
 
     // https://qlever.cs.uni-freiburg.de/osm-planet
     let endpointPlanet = $"https://qlever.cs.uni-freiburg.de/api/osm-planet"
-    let endpointGermany = $"https://qlever.cs.uni-freiburg.de/api/osm-germany"
 
+    // get wikipedia article of railway routes in germany
     let private osmWikipediaQuery () =
         """
 PREFIX osmkey: <https://www.openstreetmap.org/wiki/Key:>
+PREFIX osmrel: <https://www.openstreetmap.org/relation/>
+PREFIX ogc: <http://www.opengis.net/rdf#>
 SELECT distinct ?id ?ref ?wikipedia WHERE {
+  osmrel:51477 ogc:sfContains ?id .
   ?id osmkey:type 'route' .
   { ?id osmkey:route 'tracks' . }
   UNION
   { ?id osmkey:route 'railway' . }
   ?id osmkey:ref ?ref .
   ?id osmkey:wikipedia ?wikipedia .
-}
+} LIMIT 2000
 """
 
     // get wikipedia article via wikidata
@@ -49,7 +52,7 @@ SELECT distinct ?id ?ref ?wikidata ?wikipedia WHERE {
     ?wikipedia schema:about ?wikidata .
     ?wikipedia schema:isPartOf <https://de.wikipedia.org/> .
   }
-}
+} LIMIT 2000
 """
 
     type QleverResults = { query: string; res: (string[])[] }
@@ -58,7 +61,7 @@ SELECT distinct ?id ?ref ?wikidata ?wikipedia WHERE {
         EraKG.Request.PostAsync endpointPlanet (osmWikipediaQuery ()) format
 
     let loadWikidataArticles (format: string) : Async<string> =
-        EraKG.Request.PostAsync endpointGermany (osmWikidataQuery ()) format
+        EraKG.Request.PostAsync endpointPlanet (osmWikidataQuery ()) format
 
     let fromQueryResults (sparql: QueryResults) : Entry[] =
         sparql.results.bindings