git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2021 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.binding.fmiweather.internal.client;
  14
  15 import java.io.IOException;
  16 import java.io.StringReader;
  17 import java.math.BigDecimal;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.stream.IntStream;
  24
  25 import javax.xml.namespace.NamespaceContext;
  26 import javax.xml.parsers.DocumentBuilder;
  27 import javax.xml.parsers.DocumentBuilderFactory;
  28 import javax.xml.parsers.ParserConfigurationException;
  29 import javax.xml.xpath.XPath;
  30 import javax.xml.xpath.XPathConstants;
  31 import javax.xml.xpath.XPathExpression;
  32 import javax.xml.xpath.XPathExpressionException;
  33 import javax.xml.xpath.XPathFactory;
  34
  35 import org.eclipse.jdt.annotation.NonNullByDefault;
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.openhab.binding.fmiweather.internal.client.FMIResponse.Builder;
  38 import org.openhab.binding.fmiweather.internal.client.exception.FMIExceptionReportException;
  39 import org.openhab.binding.fmiweather.internal.client.exception.FMIIOException;
  40 import org.openhab.binding.fmiweather.internal.client.exception.FMIUnexpectedResponseException;
  41 import org.openhab.core.io.net.http.HttpUtil;
  42 import org.slf4j.Logger;
  43 import org.slf4j.LoggerFactory;
  44 import org.w3c.dom.Document;
  45 import org.w3c.dom.Node;
  46 import org.w3c.dom.NodeList;
  47 import org.xml.sax.InputSource;
  48 import org.xml.sax.SAXException;
  49
  50 /**
  51  *
  52  * Client for accessing FMI weather data
  53  *
  54  * Subject to license terms https://en.ilmatieteenlaitos.fi/open-data
  55  *
  56  *
  57  * All weather stations:
  58  * https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations&networkid=121&
  59  * Networkid parameter isexplained in entries of
  60  * https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations
  61  *
  62  * @author Sami Salonen - Initial contribution
  63  *
  64  */
  65 @NonNullByDefault
  66 public class Client {
  67
  68     private final Logger logger = LoggerFactory.getLogger(Client.class);
  69
  70     public static final String WEATHER_STATIONS_URL = "https://opendata.fmi.fi/wfs/fin?service=WFS&version=2.0.0&request=GetFeature&storedquery_id=fmi::ef::stations&networkid=121&";
  71
  72     private static final Map<String, String> NAMESPACES = new HashMap<>();
  73     static {
  74         NAMESPACES.put("target", "http://xml.fmi.fi/namespace/om/atmosphericfeatures/1.0");
  75         NAMESPACES.put("gml", "http://www.opengis.net/gml/3.2");
  76         NAMESPACES.put("xlink", "http://www.w3.org/1999/xlink");
  77         NAMESPACES.put("ows", "http://www.opengis.net/ows/1.1");
  78         NAMESPACES.put("gmlcov", "http://www.opengis.net/gmlcov/1.0");
  79         NAMESPACES.put("swe", "http://www.opengis.net/swe/2.0");
  80
  81         NAMESPACES.put("wfs", "http://www.opengis.net/wfs/2.0");
  82         NAMESPACES.put("ef", "http://inspire.ec.europa.eu/schemas/ef/4.0");
  83     }
  84     private static final NamespaceContext NAMESPACE_CONTEXT = new NamespaceContext() {
  85         @Override
  86         public @Nullable String getNamespaceURI(@Nullable String prefix) {
  87             return NAMESPACES.get(prefix);
  88         }
  89
  90         @SuppressWarnings("rawtypes")
  91         @Override
  92         public @Nullable Iterator getPrefixes(@Nullable String val) {
  93             return null;
  94         }
  95
  96         @Override
  97         public @Nullable String getPrefix(@Nullable String uri) {
  98             return null;
  99         }
 100     };
 101
 102     private DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
 103     private DocumentBuilder documentBuilder;
 104
 105     public Client() {
 106         documentBuilderFactory.setNamespaceAware(true);
 107         try {
 108             documentBuilder = documentBuilderFactory.newDocumentBuilder();
 109         } catch (ParserConfigurationException e) {
 110             throw new IllegalStateException(e);
 111         }
 112     }
 113
 114     /**
 115      * Query request and return the data
 116      *
 117      * @param request request to process
 118      * @param timeoutMillis timeout for the http call
 119      * @return data corresponding to the query
 120      * @throws FMIIOException on all I/O errors
 121      * @throws FMIUnexpectedResponseException on all unexpected content errors
 122      * @throw FMIExceptionReportException on explicit error responses from the server
 123      */
 124     public FMIResponse query(Request request, int timeoutMillis)
 125             throws FMIExceptionReportException, FMIUnexpectedResponseException, FMIIOException {
 126         try {
 127             String url = request.toUrl();
 128             String responseText = HttpUtil.executeUrl("GET", url, timeoutMillis);
 129             if (responseText == null) {
 130                 throw new FMIIOException(String.format("HTTP error with %s", request.toUrl()));
 131             }
 132             FMIResponse response = parseMultiPointCoverageXml(responseText);
 133             logger.debug("Request {} translated to url {}. Response: {}", request, url, response);
 134             return response;
 135         } catch (IOException e) {
 136             throw new FMIIOException(e);
 137         } catch (SAXException | XPathExpressionException e) {
 138             throw new FMIUnexpectedResponseException(e);
 139         }
 140     }
 141
 142     /**
 143      * Query all weather stations
 144      *
 145      * @param timeoutMillis timeout for the http call
 146      * @return locations representing stations
 147      * @throws FMIIOException on all I/O errors
 148      * @throws FMIUnexpectedResponseException on all unexpected content errors
 149      * @throw FMIExceptionReportException on explicit error responses from the server
 150      */
 151     public Set<Location> queryWeatherStations(int timeoutMillis)
 152             throws FMIIOException, FMIUnexpectedResponseException, FMIExceptionReportException {
 153         try {
 154             String response = HttpUtil.executeUrl("GET", WEATHER_STATIONS_URL, timeoutMillis);
 155             if (response == null) {
 156                 throw new FMIIOException(String.format("HTTP error with %s", WEATHER_STATIONS_URL));
 157             }
 158             return parseStations(response);
 159         } catch (IOException e) {
 160             throw new FMIIOException(e);
 161         } catch (XPathExpressionException | SAXException e) {
 162             throw new FMIUnexpectedResponseException(e);
 163         }
 164     }
 165
 166     private Set<Location> parseStations(String response) throws FMIExceptionReportException,
 167             FMIUnexpectedResponseException, SAXException, IOException, XPathExpressionException {
 168         Document document = documentBuilder.parse(new InputSource(new StringReader(response)));
 169
 170         XPath xPath = XPathFactory.newInstance().newXPath();
 171         xPath.setNamespaceContext(NAMESPACE_CONTEXT);
 172
 173         boolean isExceptionReport = ((Node) xPath.compile("/ows:ExceptionReport").evaluate(document,
 174                 XPathConstants.NODE)) != null;
 175         if (isExceptionReport) {
 176             Node exceptionCode = (Node) xPath.compile("/ows:ExceptionReport/ows:Exception/@exceptionCode")
 177                     .evaluate(document, XPathConstants.NODE);
 178             String[] exceptionText = queryNodeValues(xPath.compile("//ows:ExceptionText/text()"), document);
 179             throw new FMIExceptionReportException(exceptionCode.getNodeValue(), exceptionText);
 180         }
 181
 182         String[] fmisids = queryNodeValues(
 183                 xPath.compile(
 184                         "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/gml:identifier/text()"),
 185                 document);
 186         String[] names = queryNodeValues(xPath.compile(
 187                 "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/gml:name[@codeSpace='http://xml.fmi.fi/namespace/locationcode/name']/text()"),
 188                 document);
 189         String[] representativePoints = queryNodeValues(xPath.compile(
 190                 "/wfs:FeatureCollection/wfs:member/ef:EnvironmentalMonitoringFacility/ef:representativePoint/gml:Point/gml:pos/text()"),
 191                 document);
 192
 193         if (fmisids.length != names.length || fmisids.length != representativePoints.length) {
 194             throw new FMIUnexpectedResponseException(String.format(
 195                     "Could not all properties of locations: fmisids: %d, names: %d, representativePoints: %d",
 196                     fmisids.length, names.length, representativePoints.length));
 197         }
 198
 199         Set<Location> locations = new HashSet<>(representativePoints.length);
 200         for (int i = 0; i < representativePoints.length; i++) {
 201             BigDecimal[] latlon = parseLatLon(representativePoints[i]);
 202             locations.add(new Location(names[i], fmisids[i], latlon[0], latlon[1]));
 203         }
 204         return locations;
 205     }
 206
 207     /**
 208      * Parse FMI multipointcoverage formatted xml response
 209      *
 210      */
 211     private FMIResponse parseMultiPointCoverageXml(String response) throws FMIUnexpectedResponseException,
 212             FMIExceptionReportException, SAXException, IOException, XPathExpressionException {
 213         Document document = documentBuilder.parse(new InputSource(new StringReader(response)));
 214
 215         XPath xPath = XPathFactory.newInstance().newXPath();
 216         xPath.setNamespaceContext(NAMESPACE_CONTEXT);
 217
 218         boolean isExceptionReport = ((Node) xPath.compile("/ows:ExceptionReport").evaluate(document,
 219                 XPathConstants.NODE)) != null;
 220         if (isExceptionReport) {
 221             Node exceptionCode = (Node) xPath.compile("/ows:ExceptionReport/ows:Exception/@exceptionCode")
 222                     .evaluate(document, XPathConstants.NODE);
 223             String[] exceptionText = queryNodeValues(xPath.compile("//ows:ExceptionText/text()"), document);
 224             throw new FMIExceptionReportException(exceptionCode.getNodeValue(), exceptionText);
 225         }
 226
 227         Builder builder = new FMIResponse.Builder();
 228
 229         String[] parameters = queryNodeValues(xPath.compile("//swe:field/@name"), document);
 230         /**
 231          * Observations have FMISID (FMI Station ID?), with forecasts we use lat & lon
 232          */
 233         String[] ids = queryNodeValues(xPath.compile(
 234                 "//target:Location/gml:identifier[@codeSpace='http://xml.fmi.fi/namespace/stationcode/fmisid']/text()"),
 235                 document);
 236
 237         String[] names = queryNodeValues(xPath.compile(
 238                 "//target:Location/gml:name[@codeSpace='http://xml.fmi.fi/namespace/locationcode/name']/text()"),
 239                 document);
 240         String[] representativePointRefs = queryNodeValues(
 241                 xPath.compile("//target:Location/target:representativePoint/@xlink:href"), document);
 242
 243         if ((ids.length > 0 && ids.length != names.length) || names.length != representativePointRefs.length) {
 244             throw new FMIUnexpectedResponseException(String.format(
 245                     "Could not all properties of locations: ids: %d, names: %d, representativePointRefs: %d",
 246                     ids.length, names.length, representativePointRefs.length));
 247         }
 248
 249         Location[] locations = new Location[representativePointRefs.length];
 250         for (int i = 0; i < locations.length; i++) {
 251             BigDecimal[] latlon = findLatLon(xPath, i, document, representativePointRefs[i]);
 252             String id = ids.length == 0 ? String.format("%s,%s", latlon[0].toPlainString(), latlon[1].toPlainString())
 253                     : ids[i];
 254             locations[i] = new Location(names[i], id, latlon[0], latlon[1]);
 255         }
 256
 257         logger.trace("names ({}): {}", names.length, names);
 258         logger.trace("parameters ({}): {}", parameters.length, parameters);
 259         if (names.length == 0) {
 260             // No data, e.g. when starttime=endtime
 261             return builder.build();
 262         }
 263
 264         String latLonTimeTripletText = takeFirstOrError("positions",
 265                 queryNodeValues(xPath.compile("//gmlcov:positions/text()"), document));
 266         String[] latLonTimeTripletEntries = latLonTimeTripletText.trim().split("\\s+");
 267         logger.trace("latLonTimeTripletText: {}", latLonTimeTripletText);
 268         logger.trace("latLonTimeTripletEntries ({}): {}", latLonTimeTripletEntries.length, latLonTimeTripletEntries);
 269         int countTimestamps = latLonTimeTripletEntries.length / 3 / locations.length;
 270         long[] timestampsEpoch = IntStream.range(0, latLonTimeTripletEntries.length).filter(i -> i % 3 == 0)
 271                 .limit(countTimestamps).mapToLong(i -> Long.parseLong(latLonTimeTripletEntries[i + 2])).toArray();
 272         // Invariant
 273         assert countTimestamps == timestampsEpoch.length;
 274         logger.trace("countTimestamps ({}): {}", countTimestamps, timestampsEpoch);
 275         validatePositionEntries(locations, timestampsEpoch, latLonTimeTripletEntries);
 276
 277         String valuesText = takeFirstOrError("doubleOrNilReasonTupleList",
 278                 queryNodeValues(xPath.compile(".//gml:doubleOrNilReasonTupleList/text()"), document));
 279         String[] valuesEntries = valuesText.trim().split("\\s+");
 280         logger.trace("valuesText: {}", valuesText);
 281         logger.trace("valuesEntries ({}): {}", valuesEntries.length, valuesEntries);
 282         if (valuesEntries.length != locations.length * parameters.length * countTimestamps) {
 283             throw new FMIUnexpectedResponseException(String.format(
 284                     "Wrong number of values (%d). Expecting %d * %d * %d = %d", valuesEntries.length, locations.length,
 285                     parameters.length, countTimestamps, countTimestamps * locations.length * parameters.length));
 286         }
 287         IntStream.range(0, locations.length).forEach(locationIndex -> {
 288             for (int parameterIndex = 0; parameterIndex < parameters.length; parameterIndex++) {
 289                 for (int timestepIndex = 0; timestepIndex < countTimestamps; timestepIndex++) {
 290                     BigDecimal val = toBigDecimalOrNullIfNaN(
 291                             valuesEntries[locationIndex * countTimestamps * parameters.length
 292                                     + timestepIndex * parameters.length + parameterIndex]);
 293                     logger.trace("Found value {}={} @ time={} for location {}", parameters[parameterIndex], val,
 294                             timestampsEpoch[timestepIndex], locations[locationIndex].id);
 295                     builder.appendLocationData(locations[locationIndex], countTimestamps, parameters[parameterIndex],
 296                             timestampsEpoch[timestepIndex], val);
 297                 }
 298             }
 299         });
 300
 301         return builder.build();
 302     }
 303
 304     /**
 305      * Find representative latitude and longitude matching given xlink href attribute value
 306      *
 307      * @param xPath xpath object used for query
 308      * @param entryIndex index of the location, for logging only on errors
 309      * @param document document object
 310      * @param href xlink href attribute value. Should start with #
 311      * @return latitude and longitude values as array
 312      * @throws FMIUnexpectedResponseException parsing errors or when entry is not found
 313      * @throws XPathExpressionException xpath errors
 314      */
 315     private BigDecimal[] findLatLon(XPath xPath, int entryIndex, Document document, String href)
 316             throws FMIUnexpectedResponseException, XPathExpressionException {
 317         if (!href.startsWith("#")) {
 318             throw new FMIUnexpectedResponseException(
 319                     "Could not find valid representativePoint xlink:href, does not start with #");
 320         }
 321         String pointId = href.substring(1);
 322         String pointLatLon = takeFirstOrError(String.format("[%d]/pos", entryIndex),
 323                 queryNodeValues(xPath.compile(".//gml:Point[@gml:id='" + pointId + "']/gml:pos/text()"), document));
 324         return parseLatLon(pointLatLon);
 325     }
 326
 327     /**
 328      * Parse string reprsenting latitude longitude string separated by space
 329      *
 330      * @param pointLatLon latitude longitude string separated by space
 331      * @return latitude and longitude values as array
 332      * @throws FMIUnexpectedResponseException on parsing errors
 333      */
 334     private BigDecimal[] parseLatLon(String pointLatLon) throws FMIUnexpectedResponseException {
 335         String[] latlon = pointLatLon.split(" ");
 336         BigDecimal lat, lon;
 337         if (latlon.length != 2) {
 338             throw new FMIUnexpectedResponseException(String.format(
 339                     "Invalid latitude or longitude format, expected two values separated by space, got %d values: '%s'",
 340                     latlon.length, latlon));
 341         }
 342         try {
 343             lat = new BigDecimal(latlon[0]);
 344             lon = new BigDecimal(latlon[1]);
 345         } catch (NumberFormatException e) {
 346             throw new FMIUnexpectedResponseException(
 347                     String.format("Invalid latitude or longitude format: %s", e.getMessage()));
 348         }
 349         return new BigDecimal[] { lat, lon };
 350     }
 351
 352     private String[] queryNodeValues(XPathExpression expression, Object source) throws XPathExpressionException {
 353         NodeList nodeList = (NodeList) expression.evaluate(source, XPathConstants.NODESET);
 354         String[] values = new String[nodeList.getLength()];
 355         for (int i = 0; i < nodeList.getLength(); i++) {
 356             values[i] = nodeList.item(i).getNodeValue();
 357         }
 358         return values;
 359     }
 360
 361     /**
 362      * Asserts that length of values is exactly 1, and returns it
 363      *
 364      * @param errorDescription error description for FMIResponseException
 365      * @param values
 366      * @return
 367      * @throws FMIUnexpectedResponseException when length of values != 1
 368      */
 369     private String takeFirstOrError(String errorDescription, String[] values) throws FMIUnexpectedResponseException {
 370         if (values.length != 1) {
 371             throw new FMIUnexpectedResponseException(String.format("No unique match found: %s", errorDescription));
 372         }
 373         return values[0];
 374     }
 375
 376     /**
 377      * Convert string to BigDecimal. "NaN" string is converted to null
 378      *
 379      * @param value
 380      * @return null when value is "NaN". Otherwise BigDecimal representing the string
 381      */
 382     private @Nullable BigDecimal toBigDecimalOrNullIfNaN(String value) {
 383         if ("NaN".equals(value)) {
 384             return null;
 385         } else {
 386             return new BigDecimal(value);
 387         }
 388     }
 389
 390     /**
 391      * Validate ordering and values of gmlcov:positions (latLonTimeTripletEntries)
 392      * essentially
 393      * pos1_lat, pos1_lon, time1
 394      * pos1_lat, pos1_lon, time2
 395      * pos1_lat, pos1_lon, time3
 396      * pos2_lat, pos2_lon, time1
 397      * pos2_lat, pos2_lon, time2
 398      * ..etc..
 399      *
 400      * - lat, lon should be in correct order and match position entries ("locations")
 401      * - time should values should be exactly same for each point (above time1, time2, ...), and match given timestamps
 402      * ("timestampsEpoch")
 403      *
 404      *
 405      * @param locations previously discovered locations
 406      * @param timestampsEpoch expected timestamps
 407      * @param latLonTimeTripletEntries flat array of strings representing the array, [row1_cell1, row1_cell2,
 408      *            row2_cell1, ...]
 409      * @throws FMIUnexpectedResponseException when value ordering is not matching the expected
 410      */
 411     private void validatePositionEntries(Location[] locations, long[] timestampsEpoch,
 412             String[] latLonTimeTripletEntries) throws FMIUnexpectedResponseException {
 413         int countTimestamps = timestampsEpoch.length;
 414         for (int locationIndex = 0; locationIndex < locations.length; locationIndex++) {
 415             String firstLat = latLonTimeTripletEntries[locationIndex * countTimestamps * 3];
 416             String fistLon = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + 1];
 417
 418             // step through entries for this position
 419             for (int timestepIndex = 0; timestepIndex < countTimestamps; timestepIndex++) {
 420                 String lat = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3];
 421                 String lon = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3 + 1];
 422                 String timeEpochSec = latLonTimeTripletEntries[locationIndex * countTimestamps * 3 + timestepIndex * 3
 423                         + 2];
 424                 if (!lat.equals(firstLat) || !lon.equals(fistLon)) {
 425                     throw new FMIUnexpectedResponseException(String.format(
 426                             "positions[%d] lat, lon for time index [%d] was not matching expected ordering",
 427                             locationIndex, timestepIndex));
 428                 }
 429                 String expectedLat = locations[locationIndex].latitude.toPlainString();
 430                 String expectedLon = locations[locationIndex].longitude.toPlainString();
 431                 if (!lat.equals(expectedLat) || !lon.equals(expectedLon)) {
 432                     throw new FMIUnexpectedResponseException(String.format(
 433                             "positions[%d] lat, lon for time index [%d] was not matching representativePoint",
 434                             locationIndex, timestepIndex));
 435                 }
 436
 437                 if (Long.parseLong(timeEpochSec) != timestampsEpoch[timestepIndex]) {
 438                     throw new FMIUnexpectedResponseException(String.format(
 439                             "positions[%d] time (%s) for time index [%d] was not matching expected (%d) ordering",
 440                             locationIndex, timeEpochSec, timestepIndex, timestampsEpoch[timestepIndex]));
 441                 }
 442             }
 443         }
 444     }
 445 }