git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2024 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.binding.fmiweather.internal.client;
  14
  15 import java.io.IOException;
  16 import java.io.StringReader;
  17 import java.math.BigDecimal;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.stream.IntStream;
  24
  25 import javax.xml.namespace.NamespaceContext;
  26 import javax.xml.parsers.DocumentBuilder;
  27 import javax.xml.parsers.DocumentBuilderFactory;
  28 import javax.xml.parsers.ParserConfigurationException;
  29 import javax.xml.xpath.XPath;
  30 import javax.xml.xpath.XPathConstants;
  31 import javax.xml.xpath.XPathExpression;
  32 import javax.xml.xpath.XPathExpressionException;
  33 import javax.xml.xpath.XPathFactory;
  34
  35 import org.eclipse.jdt.annotation.NonNullByDefault;
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.openhab.binding.fmiweather.internal.client.FMIResponse.Builder;
  38 import org.openhab.binding.fmiweather.internal.client.exception.FMIExceptionReportException;
  39 import org.openhab.binding.fmiweather.internal.client.exception.FMIIOException;
  40 import org.openhab.binding.fmiweather.internal.client.exception.FMIUnexpectedResponseException;
  41 import org.openhab.core.io.net.http.HttpUtil;
  42 import org.slf4j.Logger;
  43 import org.slf4j.LoggerFactory;
  44 import org.w3c.dom.Document;
  45 import org.w3c.dom.Node;
  46 import org.w3c.dom.NodeList;
  47 import org.xml.sax.InputSource;
  48 import org.xml.sax.SAXException;
  49
  50 /**
  51  *
  52  * Client for accessing FMI weather data
  53  *
  54  * <p>
  55  * Subject to license terms https://en.ilmatieteenlaitos.fi/open-data
  56  *
  57  * @see <a href=
  58  *      "https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations&networkid=121&">
  59  *      All weather stations</a>
  60  * @see <a href=
  61  *      "https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations">
  62  *      Networkid parameter is explained in entries of</a>
  63  *
  64  * @author Sami Salonen - Initial contribution
  65  *
  66  */
  67 @NonNullByDefault
  68 public class Client {
  69
  70     private final Logger logger = LoggerFactory.getLogger(Client.class);
  71
  72     public static final String WEATHER_STATIONS_URL = "https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations&networkid=121&";
  73
  74     private static final Map<String, String> NAMESPACES = new HashMap<>();
  75     static {
  76         NAMESPACES.put("target", "http://xml.fmi.fi/namespace/om/atmosphericfeatures/1.1");
  77         NAMESPACES.put("gml", "http://www.opengis.net/gml/3.2");
  78         NAMESPACES.put("xlink", "http://www.w3.org/1999/xlink");
  79         NAMESPACES.put("ows", "http://www.opengis.net/ows/1.1");
  80         NAMESPACES.put("gmlcov", "http://www.opengis.net/gmlcov/1.0");
  81         NAMESPACES.put("swe", "http://www.opengis.net/swe/2.0");
  82
  83         NAMESPACES.put("wfs", "http://www.opengis.net/wfs/2.0");
  84         NAMESPACES.put("ef", "http://inspire.ec.europa.eu/schemas/ef/4.0");
  85     }
  86     private static final NamespaceContext NAMESPACE_CONTEXT = new NamespaceContext() {
  87         @Override
  88         public @Nullable String getNamespaceURI(@Nullable String prefix) {
  89             return NAMESPACES.get(prefix);
  90         }
  91
  92         @SuppressWarnings("rawtypes")
  93         @Override
  94         public @Nullable Iterator getPrefixes(@Nullable String val) {
  95             return null;
  96         }
  97
  98         @Override
  99         public @Nullable String getPrefix(@Nullable String uri) {
 100             return null;
 101         }
 102     };
 103
 104     private DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
 105     private DocumentBuilder documentBuilder;
 106
 107     public Client() {
 108         documentBuilderFactory.setNamespaceAware(true);
 109         try {
 110             // see https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html
 111             documentBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
 112             documentBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
 113             documentBuilderFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
 114             documentBuilderFactory.setXIncludeAware(false);
 115             documentBuilderFactory.setExpandEntityReferences(false);
 116             documentBuilder = documentBuilderFactory.newDocumentBuilder();
 117         } catch (ParserConfigurationException e) {
 118             throw new IllegalStateException(e);
 119         }
 120     }
 121
 122     /**
 123      * Query request and return the data
 124      *
 125      * @param request request to process
 126      * @param timeoutMillis timeout for the http call
 127      * @return data corresponding to the query
 128      * @throws FMIIOException on all I/O errors
 129      * @throws FMIUnexpectedResponseException on all unexpected content errors
 130      * @throws FMIExceptionReportException on explicit error responses from the server
 131      */
 132     public FMIResponse query(Request request, int timeoutMillis)
 133             throws FMIExceptionReportException, FMIUnexpectedResponseException, FMIIOException {
 134         try {
 135             String url = request.toUrl();
 136             String responseText = HttpUtil.executeUrl("GET", url, timeoutMillis);
 137             if (responseText == null) {
 138                 throw new FMIIOException(String.format("HTTP error with %s", request.toUrl()));
 139             }
 140             FMIResponse response = parseMultiPointCoverageXml(responseText);
 141             logger.debug("Request {} translated to url {}. Response: {}", request, url, response);
 142             return response;
 143         } catch (IOException e) {
 144             throw new FMIIOException(e);
 145         } catch (SAXException | XPathExpressionException e) {
 146             throw new FMIUnexpectedResponseException(e);
 147         }
 148     }
 149
 150     /**
 151      * Query all weather stations
 152      *
 153      * @param timeoutMillis timeout for the http call
 154      * @return locations representing stations
 155      * @throws FMIIOException on all I/O errors
 156      * @throws FMIUnexpectedResponseException on all unexpected content errors
 157      * @throws FMIExceptionReportException on explicit error responses from the server
 158      */
 159     public Set<Location> queryWeatherStations(int timeoutMillis)
 160             throws FMIIOException, FMIUnexpectedResponseException, FMIExceptionReportException {
 161         try {
 162             String response = HttpUtil.executeUrl("GET", WEATHER_STATIONS_URL, timeoutMillis);
 163             if (response == null) {
 164                 throw new FMIIOException(String.format("HTTP error with %s", WEATHER_STATIONS_URL));
 165             }
 166             return parseStations(response);
 167         } catch (IOException e) {
 168             throw new FMIIOException(e);
 169         } catch (XPathExpressionException | SAXException e) {
 170             throw new FMIUnexpectedResponseException(e);
 171         }
 172     }
 173
 174     private Set<Location> parseStations(String response) throws FMIExceptionReportException,
 175             FMIUnexpectedResponseException, SAXException, IOException, XPathExpressionException {
 176         Document document = documentBuilder.parse(new InputSource(new StringReader(response)));
 177
 178         XPath xPath = XPathFactory.newInstance().newXPath();
 179         xPath.setNamespaceContext(NAMESPACE_CONTEXT);
 180
 181         boolean isExceptionReport = ((Node) xPath.compile("/ows:ExceptionReport").evaluate(document,
 182                 XPathConstants.NODE)) != null;
 183         if (isExceptionReport) {
 184             Node exceptionCode = (Node) xPath.compile("/ows:ExceptionReport/ows:Exception/@exceptionCode")
 185                     .evaluate(document, XPathConstants.NODE);
 186             String[] exceptionText = queryNodeValues(xPath.compile("//ows:ExceptionText/text()"), document);
 187             throw new FMIExceptionReportException(exceptionCode.getNodeValue(), exceptionText);
 188         }
 189
 190         String[] fmisids = queryNodeValues(
 191                 xPath.compile(
 192                         "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/gml:identifier/text()"),
 193                 document);
 194         String[] names = queryNodeValues(xPath.compile(
 195                 "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/gml:name[@codeSpace='http://xml.fmi.fi/namespace/locationcode/name']/text()"),
 196                 document);
 197         String[] representativePoints = queryNodeValues(xPath.compile(
 198                 "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/ef:representativePoint/gml:Point/gml:pos/text()"),
 199                 document);
 200
 201         if (fmisids.length != names.length || fmisids.length != representativePoints.length) {
 202             throw new FMIUnexpectedResponseException(String.format(
 203                     "Could not all properties of locations: fmisids: %d, names: %d, representativePoints: %d",
 204                     fmisids.length, names.length, representativePoints.length));
 205         }
 206
 207         Set<Location> locations = new HashSet<>(representativePoints.length);
 208         for (int i = 0; i < representativePoints.length; i++) {
 209             BigDecimal[] latlon = parseLatLon(representativePoints[i]);
 210             locations.add(new Location(names[i], fmisids[i], latlon[0], latlon[1]));
 211         }
 212         return locations;
 213     }
 214
 215     /**
 216      * Parse FMI multipointcoverage formatted xml response
 217      *
 218      */
 219     private FMIResponse parseMultiPointCoverageXml(String response) throws FMIUnexpectedResponseException,
 220             FMIExceptionReportException, SAXException, IOException, XPathExpressionException {
 221         Document document = documentBuilder.parse(new InputSource(new StringReader(response)));
 222
 223         XPath xPath = XPathFactory.newInstance().newXPath();
 224         xPath.setNamespaceContext(NAMESPACE_CONTEXT);
 225
 226         boolean isExceptionReport = ((Node) xPath.compile("/ows:ExceptionReport").evaluate(document,
 227                 XPathConstants.NODE)) != null;
 228         if (isExceptionReport) {
 229             Node exceptionCode = (Node) xPath.compile("/ows:ExceptionReport/ows:Exception/@exceptionCode")
 230                     .evaluate(document, XPathConstants.NODE);
 231             String[] exceptionText = queryNodeValues(xPath.compile("//ows:ExceptionText/text()"), document);
 232             throw new FMIExceptionReportException(exceptionCode.getNodeValue(), exceptionText);
 233         }
 234
 235         Builder builder = new FMIResponse.Builder();
 236
 237         String[] parameters = queryNodeValues(xPath.compile("//swe:field/@name"), document);
 238         /**
 239          * Observations have FMISID (FMI Station ID?), with forecasts we use lat & lon
 240          */
 241         String[] ids = queryNodeValues(xPath.compile(
 242                 "//target:Location/gml:identifier[@codeSpace='http://xml.fmi.fi/namespace/stationcode/fmisid']/text()"),
 243                 document);
 244
 245         String[] names = queryNodeValues(xPath.compile(
 246                 "//target:Location/gml:name[@codeSpace='http://xml.fmi.fi/namespace/locationcode/name']/text()"),
 247                 document);
 248         String[] representativePointRefs = queryNodeValues(
 249                 xPath.compile("//target:Location/target:representativePoint/@xlink:href"), document);
 250
 251         if ((ids.length > 0 && ids.length != names.length) || names.length != representativePointRefs.length) {
 252             throw new FMIUnexpectedResponseException(String.format(
 253                     "Could not all properties of locations: ids: %d, names: %d, representativePointRefs: %d",
 254                     ids.length, names.length, representativePointRefs.length));
 255         }
 256
 257         Location[] locations = new Location[representativePointRefs.length];
 258         for (int i = 0; i < locations.length; i++) {
 259             BigDecimal[] latlon = findLatLon(xPath, i, document, representativePointRefs[i]);
 260             String id = ids.length == 0 ? String.format("%s,%s", latlon[0].toPlainString(), latlon[1].toPlainString())
 261                     : ids[i];
 262             locations[i] = new Location(names[i], id, latlon[0], latlon[1]);
 263         }
 264
 265         logger.trace("names ({}): {}", names.length, names);
 266         logger.trace("parameters ({}): {}", parameters.length, parameters);
 267         if (names.length == 0) {
 268             // No data, e.g. when starttime=endtime
 269             return builder.build();
 270         }
 271
 272         String latLonTimeTripletText = takeFirstOrError("positions",
 273                 queryNodeValues(xPath.compile("//gmlcov:positions/text()"), document));
 274         String[] latLonTimeTripletEntries = latLonTimeTripletText.trim().split("\\s+");
 275         logger.trace("latLonTimeTripletText: {}", latLonTimeTripletText);
 276         logger.trace("latLonTimeTripletEntries ({}): {}", latLonTimeTripletEntries.length, latLonTimeTripletEntries);
 277         int countTimestamps = latLonTimeTripletEntries.length / 3 / locations.length;
 278         long[] timestampsEpoch = IntStream.range(0, latLonTimeTripletEntries.length).filter(i -> i % 3 == 0)
 279                 .limit(countTimestamps).mapToLong(i -> Long.parseLong(latLonTimeTripletEntries[i + 2])).toArray();
 280         // Invariant
 281         assert countTimestamps == timestampsEpoch.length;
 282         logger.trace("countTimestamps ({}): {}", countTimestamps, timestampsEpoch);
 283         validatePositionEntries(locations, timestampsEpoch, latLonTimeTripletEntries);
 284
 285         String valuesText = takeFirstOrError("doubleOrNilReasonTupleList",
 286                 queryNodeValues(xPath.compile(".//gml:doubleOrNilReasonTupleList/text()"), document));
 287         String[] valuesEntries = valuesText.trim().split("\\s+");
 288         logger.trace("valuesText: {}", valuesText);
 289         logger.trace("valuesEntries ({}): {}", valuesEntries.length, valuesEntries);
 290         if (valuesEntries.length != locations.length * parameters.length * countTimestamps) {
 291             throw new FMIUnexpectedResponseException(String.format(
 292                     "Wrong number of values (%d). Expecting %d * %d * %d = %d", valuesEntries.length, locations.length,
 293                     parameters.length, countTimestamps, countTimestamps * locations.length * parameters.length));
 294         }
 295         IntStream.range(0, locations.length).forEach(locationIndex -> {
 296             for (int parameterIndex = 0; parameterIndex < parameters.length; parameterIndex++) {
 297                 for (int timestepIndex = 0; timestepIndex < countTimestamps; timestepIndex++) {
 298                     BigDecimal val = toBigDecimalOrNullIfNaN(
 299                             valuesEntries[locationIndex * countTimestamps * parameters.length
 300                                     + timestepIndex * parameters.length + parameterIndex]);
 301                     logger.trace("Found value {}={} @ time={} for location {}", parameters[parameterIndex], val,
 302                             timestampsEpoch[timestepIndex], locations[locationIndex].id);
 303                     builder.appendLocationData(locations[locationIndex], countTimestamps, parameters[parameterIndex],
 304                             timestampsEpoch[timestepIndex], val);
 305                 }
 306             }
 307         });
 308
 309         return builder.build();
 310     }
 311
 312     /**
 313      * Find representative latitude and longitude matching given xlink href attribute value
 314      *
 315      * @param xPath xpath object used for query
 316      * @param entryIndex index of the location, for logging only on errors
 317      * @param document document object
 318      * @param href xlink href attribute value. Should start with #
 319      * @return latitude and longitude values as array
 320      * @throws FMIUnexpectedResponseException parsing errors or when entry is not found
 321      * @throws XPathExpressionException xpath errors
 322      */
 323     private BigDecimal[] findLatLon(XPath xPath, int entryIndex, Document document, String href)
 324             throws FMIUnexpectedResponseException, XPathExpressionException {
 325         if (!href.startsWith("#")) {
 326             throw new FMIUnexpectedResponseException(
 327                     "Could not find valid representativePoint xlink:href, does not start with #");
 328         }
 329         String pointId = href.substring(1);
 330         String pointLatLon = takeFirstOrError(String.format("[%d]/pos", entryIndex),
 331                 queryNodeValues(xPath.compile(".//gml:Point[@gml:id='" + pointId + "']/gml:pos/text()"), document));
 332         return parseLatLon(pointLatLon);
 333     }
 334
 335     /**
 336      * Parse string reprsenting latitude longitude string separated by space
 337      *
 338      * @param pointLatLon latitude longitude string separated by space
 339      * @return latitude and longitude values as array
 340      * @throws FMIUnexpectedResponseException on parsing errors
 341      */
 342     private BigDecimal[] parseLatLon(String pointLatLon) throws FMIUnexpectedResponseException {
 343         String[] latlon = pointLatLon.split(" ");
 344         BigDecimal lat, lon;
 345         if (latlon.length != 2) {
 346             throw new FMIUnexpectedResponseException(String.format(
 347                     "Invalid latitude or longitude format, expected two values separated by space, got %d values: '%s'",
 348                     latlon.length, latlon));
 349         }
 350         try {
 351             lat = new BigDecimal(latlon[0]);
 352             lon = new BigDecimal(latlon[1]);
 353         } catch (NumberFormatException e) {
 354             throw new FMIUnexpectedResponseException(
 355                     String.format("Invalid latitude or longitude format: %s", e.getMessage()));
 356         }
 357         return new BigDecimal[] { lat, lon };
 358     }
 359
 360     private String[] queryNodeValues(XPathExpression expression, Object source) throws XPathExpressionException {
 361         NodeList nodeList = (NodeList) expression.evaluate(source, XPathConstants.NODESET);
 362         String[] values = new String[nodeList.getLength()];
 363         for (int i = 0; i < nodeList.getLength(); i++) {
 364             values[i] = nodeList.item(i).getNodeValue();
 365         }
 366         return values;
 367     }
 368
 369     /**
 370      * Asserts that length of values is exactly 1, and returns it
 371      *
 372      * @param errorDescription error description for FMIResponseException
 373      * @param values
 374      * @return
 375      * @throws FMIUnexpectedResponseException when length of values != 1
 376      */
 377     private String takeFirstOrError(String errorDescription, String[] values) throws FMIUnexpectedResponseException {
 378         if (values.length != 1) {
 379             throw new FMIUnexpectedResponseException(
 380                     String.format("No unique match found: %s (found %d)", errorDescription, values.length));
 381         }
 382         return values[0];
 383     }
 384
 385     /**
 386      * Convert string to BigDecimal. "NaN" string is converted to null
 387      *
 388      * @param value
 389      * @return null when value is "NaN". Otherwise BigDecimal representing the string
 390      */
 391     private @Nullable BigDecimal toBigDecimalOrNullIfNaN(String value) {
 392         if ("NaN".equals(value)) {
 393             return null;
 394         } else {
 395             return new BigDecimal(value);
 396         }
 397     }
 398
 399     /**
 400      * Validate ordering and values of gmlcov:positions (latLonTimeTripletEntries)
 401      * essentially
 402      * pos1_lat, pos1_lon, time1
 403      * pos1_lat, pos1_lon, time2
 404      * pos1_lat, pos1_lon, time3
 405      * pos2_lat, pos2_lon, time1
 406      * pos2_lat, pos2_lon, time2
 407      * ..etc..
 408      *
 409      * - lat, lon should be in correct order and match position entries ("locations")
 410      * - time should values should be exactly same for each point (above time1, time2, ...), and match given timestamps
 411      * ("timestampsEpoch")
 412      *
 413      *
 414      * @param locations previously discovered locations
 415      * @param timestampsEpoch expected timestamps
 416      * @param latLonTimeTripletEntries flat array of strings representing the array, [row1_cell1, row1_cell2,
 417      *            row2_cell1, ...]
 418      * @throws FMIUnexpectedResponseException when value ordering is not matching the expected
 419      */
 420     private void validatePositionEntries(Location[] locations, long[] timestampsEpoch,
 421             String[] latLonTimeTripletEntries) throws FMIUnexpectedResponseException {
 422         int countTimestamps = timestampsEpoch.length;
 423         for (int locationIndex = 0; locationIndex < locations.length; locationIndex++) {
 424             String firstLat = latLonTimeTripletEntries[locationIndex * countTimestamps * 3];
 425             String fistLon = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + 1];
 426
 427             // step through entries for this position
 428             for (int timestepIndex = 0; timestepIndex < countTimestamps; timestepIndex++) {
 429                 String lat = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3];
 430                 String lon = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3 + 1];
 431                 String timeEpochSec = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3
 432                         + 2];
 433                 if (!lat.equals(firstLat) || !lon.equals(fistLon)) {
 434                     throw new FMIUnexpectedResponseException(String.format(
 435                             "positions[%d] lat, lon for time index [%d] was not matching expected ordering",
 436                             locationIndex, timestepIndex));
 437                 }
 438                 String expectedLat = locations[locationIndex].latitude.toPlainString();
 439                 String expectedLon = locations[locationIndex].longitude.toPlainString();
 440                 if (!lat.equals(expectedLat) || !lon.equals(expectedLon)) {
 441                     throw new FMIUnexpectedResponseException(String.format(
 442                             "positions[%d] lat, lon for time index [%d] was not matching representativePoint",
 443                             locationIndex, timestepIndex));
 444                 }
 445
 446                 if (Long.parseLong(timeEpochSec) != timestampsEpoch[timestepIndex]) {
 447                     throw new FMIUnexpectedResponseException(String.format(
 448                             "positions[%d] time (%s) for time index [%d] was not matching expected (%d) ordering",
 449                             locationIndex, timeEpochSec, timestepIndex, timestampsEpoch[timestepIndex]));
 450                 }
 451             }
 452         }
 453     }
 454 }