git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2023 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.binding.fmiweather.internal.client;
  14
  15 import java.io.IOException;
  16 import java.io.StringReader;
  17 import java.math.BigDecimal;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.stream.IntStream;
  24
  25 import javax.xml.namespace.NamespaceContext;
  26 import javax.xml.parsers.DocumentBuilder;
  27 import javax.xml.parsers.DocumentBuilderFactory;
  28 import javax.xml.parsers.ParserConfigurationException;
  29 import javax.xml.xpath.XPath;
  30 import javax.xml.xpath.XPathConstants;
  31 import javax.xml.xpath.XPathExpression;
  32 import javax.xml.xpath.XPathExpressionException;
  33 import javax.xml.xpath.XPathFactory;
  34
  35 import org.eclipse.jdt.annotation.NonNullByDefault;
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.openhab.binding.fmiweather.internal.client.FMIResponse.Builder;
  38 import org.openhab.binding.fmiweather.internal.client.exception.FMIExceptionReportException;
  39 import org.openhab.binding.fmiweather.internal.client.exception.FMIIOException;
  40 import org.openhab.binding.fmiweather.internal.client.exception.FMIUnexpectedResponseException;
  41 import org.openhab.core.io.net.http.HttpUtil;
  42 import org.slf4j.Logger;
  43 import org.slf4j.LoggerFactory;
  44 import org.w3c.dom.Document;
  45 import org.w3c.dom.Node;
  46 import org.w3c.dom.NodeList;
  47 import org.xml.sax.InputSource;
  48 import org.xml.sax.SAXException;
  49
  50 /**
  51  *
  52  * Client for accessing FMI weather data
  53  *
  54  * Subject to license terms https://en.ilmatieteenlaitos.fi/open-data
  55  *
  56  *
  57  * All weather stations:
  58  * https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations&networkid=121&
  59  * Networkid parameter isexplained in entries of
  60  * https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations
  61  *
  62  * @author Sami Salonen - Initial contribution
  63  *
  64  */
  65 @NonNullByDefault
  66 public class Client {
  67
  68     private final Logger logger = LoggerFactory.getLogger(Client.class);
  69
  70     public static final String WEATHER_STATIONS_URL = "https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations&networkid=121&";
  71
  72     private static final Map<String, String> NAMESPACES = new HashMap<>();
  73     static {
  74         NAMESPACES.put("target", "http://xml.fmi.fi/namespace/om/atmosphericfeatures/1.1");
  75         NAMESPACES.put("gml", "http://www.opengis.net/gml/3.2");
  76         NAMESPACES.put("xlink", "http://www.w3.org/1999/xlink");
  77         NAMESPACES.put("ows", "http://www.opengis.net/ows/1.1");
  78         NAMESPACES.put("gmlcov", "http://www.opengis.net/gmlcov/1.0");
  79         NAMESPACES.put("swe", "http://www.opengis.net/swe/2.0");
  80
  81         NAMESPACES.put("wfs", "http://www.opengis.net/wfs/2.0");
  82         NAMESPACES.put("ef", "http://inspire.ec.europa.eu/schemas/ef/4.0");
  83     }
  84     private static final NamespaceContext NAMESPACE_CONTEXT = new NamespaceContext() {
  85         @Override
  86         public @Nullable String getNamespaceURI(@Nullable String prefix) {
  87             return NAMESPACES.get(prefix);
  88         }
  89
  90         @SuppressWarnings("rawtypes")
  91         @Override
  92         public @Nullable Iterator getPrefixes(@Nullable String val) {
  93             return null;
  94         }
  95
  96         @Override
  97         public @Nullable String getPrefix(@Nullable String uri) {
  98             return null;
  99         }
 100     };
 101
 102     private DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
 103     private DocumentBuilder documentBuilder;
 104
 105     public Client() {
 106         documentBuilderFactory.setNamespaceAware(true);
 107         try {
 108             // see https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html
 109             documentBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
 110             documentBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
 111             documentBuilderFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
 112             documentBuilderFactory.setXIncludeAware(false);
 113             documentBuilderFactory.setExpandEntityReferences(false);
 114             documentBuilder = documentBuilderFactory.newDocumentBuilder();
 115         } catch (ParserConfigurationException e) {
 116             throw new IllegalStateException(e);
 117         }
 118     }
 119
 120     /**
 121      * Query request and return the data
 122      *
 123      * @param request request to process
 124      * @param timeoutMillis timeout for the http call
 125      * @return data corresponding to the query
 126      * @throws FMIIOException on all I/O errors
 127      * @throws FMIUnexpectedResponseException on all unexpected content errors
 128      * @throw FMIExceptionReportException on explicit error responses from the server
 129      */
 130     public FMIResponse query(Request request, int timeoutMillis)
 131             throws FMIExceptionReportException, FMIUnexpectedResponseException, FMIIOException {
 132         try {
 133             String url = request.toUrl();
 134             String responseText = HttpUtil.executeUrl("GET", url, timeoutMillis);
 135             if (responseText == null) {
 136                 throw new FMIIOException(String.format("HTTP error with %s", request.toUrl()));
 137             }
 138             FMIResponse response = parseMultiPointCoverageXml(responseText);
 139             logger.debug("Request {} translated to url {}. Response: {}", request, url, response);
 140             return response;
 141         } catch (IOException e) {
 142             throw new FMIIOException(e);
 143         } catch (SAXException | XPathExpressionException e) {
 144             throw new FMIUnexpectedResponseException(e);
 145         }
 146     }
 147
 148     /**
 149      * Query all weather stations
 150      *
 151      * @param timeoutMillis timeout for the http call
 152      * @return locations representing stations
 153      * @throws FMIIOException on all I/O errors
 154      * @throws FMIUnexpectedResponseException on all unexpected content errors
 155      * @throw FMIExceptionReportException on explicit error responses from the server
 156      */
 157     public Set<Location> queryWeatherStations(int timeoutMillis)
 158             throws FMIIOException, FMIUnexpectedResponseException, FMIExceptionReportException {
 159         try {
 160             String response = HttpUtil.executeUrl("GET", WEATHER_STATIONS_URL, timeoutMillis);
 161             if (response == null) {
 162                 throw new FMIIOException(String.format("HTTP error with %s", WEATHER_STATIONS_URL));
 163             }
 164             return parseStations(response);
 165         } catch (IOException e) {
 166             throw new FMIIOException(e);
 167         } catch (XPathExpressionException | SAXException e) {
 168             throw new FMIUnexpectedResponseException(e);
 169         }
 170     }
 171
 172     private Set<Location> parseStations(String response) throws FMIExceptionReportException,
 173             FMIUnexpectedResponseException, SAXException, IOException, XPathExpressionException {
 174         Document document = documentBuilder.parse(new InputSource(new StringReader(response)));
 175
 176         XPath xPath = XPathFactory.newInstance().newXPath();
 177         xPath.setNamespaceContext(NAMESPACE_CONTEXT);
 178
 179         boolean isExceptionReport = ((Node) xPath.compile("/ows:ExceptionReport").evaluate(document,
 180                 XPathConstants.NODE)) != null;
 181         if (isExceptionReport) {
 182             Node exceptionCode = (Node) xPath.compile("/ows:ExceptionReport/ows:Exception/@exceptionCode")
 183                     .evaluate(document, XPathConstants.NODE);
 184             String[] exceptionText = queryNodeValues(xPath.compile("//ows:ExceptionText/text()"), document);
 185             throw new FMIExceptionReportException(exceptionCode.getNodeValue(), exceptionText);
 186         }
 187
 188         String[] fmisids = queryNodeValues(
 189                 xPath.compile(
 190                         "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/gml:identifier/text()"),
 191                 document);
 192         String[] names = queryNodeValues(xPath.compile(
 193                 "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/gml:name[@codeSpace='http://xml.fmi.fi/namespace/locationcode/name']/text()"),
 194                 document);
 195         String[] representativePoints = queryNodeValues(xPath.compile(
 196                 "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/ef:representativePoint/gml:Point/gml:pos/text()"),
 197                 document);
 198
 199         if (fmisids.length != names.length || fmisids.length != representativePoints.length) {
 200             throw new FMIUnexpectedResponseException(String.format(
 201                     "Could not all properties of locations: fmisids: %d, names: %d, representativePoints: %d",
 202                     fmisids.length, names.length, representativePoints.length));
 203         }
 204
 205         Set<Location> locations = new HashSet<>(representativePoints.length);
 206         for (int i = 0; i < representativePoints.length; i++) {
 207             BigDecimal[] latlon = parseLatLon(representativePoints[i]);
 208             locations.add(new Location(names[i], fmisids[i], latlon[0], latlon[1]));
 209         }
 210         return locations;
 211     }
 212
 213     /**
 214      * Parse FMI multipointcoverage formatted xml response
 215      *
 216      */
 217     private FMIResponse parseMultiPointCoverageXml(String response) throws FMIUnexpectedResponseException,
 218             FMIExceptionReportException, SAXException, IOException, XPathExpressionException {
 219         Document document = documentBuilder.parse(new InputSource(new StringReader(response)));
 220
 221         XPath xPath = XPathFactory.newInstance().newXPath();
 222         xPath.setNamespaceContext(NAMESPACE_CONTEXT);
 223
 224         boolean isExceptionReport = ((Node) xPath.compile("/ows:ExceptionReport").evaluate(document,
 225                 XPathConstants.NODE)) != null;
 226         if (isExceptionReport) {
 227             Node exceptionCode = (Node) xPath.compile("/ows:ExceptionReport/ows:Exception/@exceptionCode")
 228                     .evaluate(document, XPathConstants.NODE);
 229             String[] exceptionText = queryNodeValues(xPath.compile("//ows:ExceptionText/text()"), document);
 230             throw new FMIExceptionReportException(exceptionCode.getNodeValue(), exceptionText);
 231         }
 232
 233         Builder builder = new FMIResponse.Builder();
 234
 235         String[] parameters = queryNodeValues(xPath.compile("//swe:field/@name"), document);
 236         /**
 237          * Observations have FMISID (FMI Station ID?), with forecasts we use lat & lon
 238          */
 239         String[] ids = queryNodeValues(xPath.compile(
 240                 "//target:Location/gml:identifier[@codeSpace='http://xml.fmi.fi/namespace/stationcode/fmisid']/text()"),
 241                 document);
 242
 243         String[] names = queryNodeValues(xPath.compile(
 244                 "//target:Location/gml:name[@codeSpace='http://xml.fmi.fi/namespace/locationcode/name']/text()"),
 245                 document);
 246         String[] representativePointRefs = queryNodeValues(
 247                 xPath.compile("//target:Location/target:representativePoint/@xlink:href"), document);
 248
 249         if ((ids.length > 0 && ids.length != names.length) || names.length != representativePointRefs.length) {
 250             throw new FMIUnexpectedResponseException(String.format(
 251                     "Could not all properties of locations: ids: %d, names: %d, representativePointRefs: %d",
 252                     ids.length, names.length, representativePointRefs.length));
 253         }
 254
 255         Location[] locations = new Location[representativePointRefs.length];
 256         for (int i = 0; i < locations.length; i++) {
 257             BigDecimal[] latlon = findLatLon(xPath, i, document, representativePointRefs[i]);
 258             String id = ids.length == 0 ? String.format("%s,%s", latlon[0].toPlainString(), latlon[1].toPlainString())
 259                     : ids[i];
 260             locations[i] = new Location(names[i], id, latlon[0], latlon[1]);
 261         }
 262
 263         logger.trace("names ({}): {}", names.length, names);
 264         logger.trace("parameters ({}): {}", parameters.length, parameters);
 265         if (names.length == 0) {
 266             // No data, e.g. when starttime=endtime
 267             return builder.build();
 268         }
 269
 270         String latLonTimeTripletText = takeFirstOrError("positions",
 271                 queryNodeValues(xPath.compile("//gmlcov:positions/text()"), document));
 272         String[] latLonTimeTripletEntries = latLonTimeTripletText.trim().split("\\s+");
 273         logger.trace("latLonTimeTripletText: {}", latLonTimeTripletText);
 274         logger.trace("latLonTimeTripletEntries ({}): {}", latLonTimeTripletEntries.length, latLonTimeTripletEntries);
 275         int countTimestamps = latLonTimeTripletEntries.length / 3 / locations.length;
 276         long[] timestampsEpoch = IntStream.range(0, latLonTimeTripletEntries.length).filter(i -> i % 3 == 0)
 277                 .limit(countTimestamps).mapToLong(i -> Long.parseLong(latLonTimeTripletEntries[i + 2])).toArray();
 278         // Invariant
 279         assert countTimestamps == timestampsEpoch.length;
 280         logger.trace("countTimestamps ({}): {}", countTimestamps, timestampsEpoch);
 281         validatePositionEntries(locations, timestampsEpoch, latLonTimeTripletEntries);
 282
 283         String valuesText = takeFirstOrError("doubleOrNilReasonTupleList",
 284                 queryNodeValues(xPath.compile(".//gml:doubleOrNilReasonTupleList/text()"), document));
 285         String[] valuesEntries = valuesText.trim().split("\\s+");
 286         logger.trace("valuesText: {}", valuesText);
 287         logger.trace("valuesEntries ({}): {}", valuesEntries.length, valuesEntries);
 288         if (valuesEntries.length != locations.length * parameters.length * countTimestamps) {
 289             throw new FMIUnexpectedResponseException(String.format(
 290                     "Wrong number of values (%d). Expecting %d * %d * %d = %d", valuesEntries.length, locations.length,
 291                     parameters.length, countTimestamps, countTimestamps * locations.length * parameters.length));
 292         }
 293         IntStream.range(0, locations.length).forEach(locationIndex -> {
 294             for (int parameterIndex = 0; parameterIndex < parameters.length; parameterIndex++) {
 295                 for (int timestepIndex = 0; timestepIndex < countTimestamps; timestepIndex++) {
 296                     BigDecimal val = toBigDecimalOrNullIfNaN(
 297                             valuesEntries[locationIndex * countTimestamps * parameters.length
 298                                     + timestepIndex * parameters.length + parameterIndex]);
 299                     logger.trace("Found value {}={} @ time={} for location {}", parameters[parameterIndex], val,
 300                             timestampsEpoch[timestepIndex], locations[locationIndex].id);
 301                     builder.appendLocationData(locations[locationIndex], countTimestamps, parameters[parameterIndex],
 302                             timestampsEpoch[timestepIndex], val);
 303                 }
 304             }
 305         });
 306
 307         return builder.build();
 308     }
 309
 310     /**
 311      * Find representative latitude and longitude matching given xlink href attribute value
 312      *
 313      * @param xPath xpath object used for query
 314      * @param entryIndex index of the location, for logging only on errors
 315      * @param document document object
 316      * @param href xlink href attribute value. Should start with #
 317      * @return latitude and longitude values as array
 318      * @throws FMIUnexpectedResponseException parsing errors or when entry is not found
 319      * @throws XPathExpressionException xpath errors
 320      */
 321     private BigDecimal[] findLatLon(XPath xPath, int entryIndex, Document document, String href)
 322             throws FMIUnexpectedResponseException, XPathExpressionException {
 323         if (!href.startsWith("#")) {
 324             throw new FMIUnexpectedResponseException(
 325                     "Could not find valid representativePoint xlink:href, does not start with #");
 326         }
 327         String pointId = href.substring(1);
 328         String pointLatLon = takeFirstOrError(String.format("[%d]/pos", entryIndex),
 329                 queryNodeValues(xPath.compile(".//gml:Point[@gml:id='" + pointId + "']/gml:pos/text()"), document));
 330         return parseLatLon(pointLatLon);
 331     }
 332
 333     /**
 334      * Parse string reprsenting latitude longitude string separated by space
 335      *
 336      * @param pointLatLon latitude longitude string separated by space
 337      * @return latitude and longitude values as array
 338      * @throws FMIUnexpectedResponseException on parsing errors
 339      */
 340     private BigDecimal[] parseLatLon(String pointLatLon) throws FMIUnexpectedResponseException {
 341         String[] latlon = pointLatLon.split(" ");
 342         BigDecimal lat, lon;
 343         if (latlon.length != 2) {
 344             throw new FMIUnexpectedResponseException(String.format(
 345                     "Invalid latitude or longitude format, expected two values separated by space, got %d values: '%s'",
 346                     latlon.length, latlon));
 347         }
 348         try {
 349             lat = new BigDecimal(latlon[0]);
 350             lon = new BigDecimal(latlon[1]);
 351         } catch (NumberFormatException e) {
 352             throw new FMIUnexpectedResponseException(
 353                     String.format("Invalid latitude or longitude format: %s", e.getMessage()));
 354         }
 355         return new BigDecimal[] { lat, lon };
 356     }
 357
 358     private String[] queryNodeValues(XPathExpression expression, Object source) throws XPathExpressionException {
 359         NodeList nodeList = (NodeList) expression.evaluate(source, XPathConstants.NODESET);
 360         String[] values = new String[nodeList.getLength()];
 361         for (int i = 0; i < nodeList.getLength(); i++) {
 362             values[i] = nodeList.item(i).getNodeValue();
 363         }
 364         return values;
 365     }
 366
 367     /**
 368      * Asserts that length of values is exactly 1, and returns it
 369      *
 370      * @param errorDescription error description for FMIResponseException
 371      * @param values
 372      * @return
 373      * @throws FMIUnexpectedResponseException when length of values != 1
 374      */
 375     private String takeFirstOrError(String errorDescription, String[] values) throws FMIUnexpectedResponseException {
 376         if (values.length != 1) {
 377             throw new FMIUnexpectedResponseException(
 378                     String.format("No unique match found: %s (found %d)", errorDescription, values.length));
 379         }
 380         return values[0];
 381     }
 382
 383     /**
 384      * Convert string to BigDecimal. "NaN" string is converted to null
 385      *
 386      * @param value
 387      * @return null when value is "NaN". Otherwise BigDecimal representing the string
 388      */
 389     private @Nullable BigDecimal toBigDecimalOrNullIfNaN(String value) {
 390         if ("NaN".equals(value)) {
 391             return null;
 392         } else {
 393             return new BigDecimal(value);
 394         }
 395     }
 396
 397     /**
 398      * Validate ordering and values of gmlcov:positions (latLonTimeTripletEntries)
 399      * essentially
 400      * pos1_lat, pos1_lon, time1
 401      * pos1_lat, pos1_lon, time2
 402      * pos1_lat, pos1_lon, time3
 403      * pos2_lat, pos2_lon, time1
 404      * pos2_lat, pos2_lon, time2
 405      * ..etc..
 406      *
 407      * - lat, lon should be in correct order and match position entries ("locations")
 408      * - time should values should be exactly same for each point (above time1, time2, ...), and match given timestamps
 409      * ("timestampsEpoch")
 410      *
 411      *
 412      * @param locations previously discovered locations
 413      * @param timestampsEpoch expected timestamps
 414      * @param latLonTimeTripletEntries flat array of strings representing the array, [row1_cell1, row1_cell2,
 415      *            row2_cell1, ...]
 416      * @throws FMIUnexpectedResponseException when value ordering is not matching the expected
 417      */
 418     private void validatePositionEntries(Location[] locations, long[] timestampsEpoch,
 419             String[] latLonTimeTripletEntries) throws FMIUnexpectedResponseException {
 420         int countTimestamps = timestampsEpoch.length;
 421         for (int locationIndex = 0; locationIndex < locations.length; locationIndex++) {
 422             String firstLat = latLonTimeTripletEntries[locationIndex * countTimestamps * 3];
 423             String fistLon = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + 1];
 424
 425             // step through entries for this position
 426             for (int timestepIndex = 0; timestepIndex < countTimestamps; timestepIndex++) {
 427                 String lat = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3];
 428                 String lon = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3 + 1];
 429                 String timeEpochSec = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3
 430                         + 2];
 431                 if (!lat.equals(firstLat) || !lon.equals(fistLon)) {
 432                     throw new FMIUnexpectedResponseException(String.format(
 433                             "positions[%d] lat, lon for time index [%d] was not matching expected ordering",
 434                             locationIndex, timestepIndex));
 435                 }
 436                 String expectedLat = locations[locationIndex].latitude.toPlainString();
 437                 String expectedLon = locations[locationIndex].longitude.toPlainString();
 438                 if (!lat.equals(expectedLat) || !lon.equals(expectedLon)) {
 439                     throw new FMIUnexpectedResponseException(String.format(
 440                             "positions[%d] lat, lon for time index [%d] was not matching representativePoint",
 441                             locationIndex, timestepIndex));
 442                 }
 443
 444                 if (Long.parseLong(timeEpochSec) != timestampsEpoch[timestepIndex]) {
 445                     throw new FMIUnexpectedResponseException(String.format(
 446                             "positions[%d] time (%s) for time index [%d] was not matching expected (%d) ordering",
 447                             locationIndex, timeEpochSec, timestepIndex, timestampsEpoch[timestepIndex]));
 448                 }
 449             }
 450         }
 451     }
 452 }