2 * Copyright (c) 2010-2022 Contributors to the openHAB project
4 * See the NOTICE file(s) distributed with this work for additional
7 * This program and the accompanying materials are made available under the
8 * terms of the Eclipse Public License 2.0 which is available at
9 * http://www.eclipse.org/legal/epl-2.0
11 * SPDX-License-Identifier: EPL-2.0
13 package org.openhab.persistence.rrd4j.internal;
15 import java.io.IOException;
16 import java.nio.file.Files;
17 import java.nio.file.Path;
18 import java.time.Instant;
19 import java.time.ZoneId;
20 import java.time.ZonedDateTime;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.HashMap;
24 import java.util.Iterator;
25 import java.util.List;
26 import java.util.Locale;
29 import java.util.concurrent.ConcurrentHashMap;
30 import java.util.concurrent.Executors;
31 import java.util.concurrent.RejectedExecutionException;
32 import java.util.concurrent.ScheduledExecutorService;
33 import java.util.concurrent.ScheduledFuture;
34 import java.util.concurrent.TimeUnit;
36 import javax.measure.Quantity;
37 import javax.measure.Unit;
39 import org.eclipse.jdt.annotation.NonNullByDefault;
40 import org.eclipse.jdt.annotation.Nullable;
41 import org.openhab.core.OpenHAB;
42 import org.openhab.core.common.NamedThreadFactory;
43 import org.openhab.core.items.GroupItem;
44 import org.openhab.core.items.Item;
45 import org.openhab.core.items.ItemNotFoundException;
46 import org.openhab.core.items.ItemRegistry;
47 import org.openhab.core.items.ItemUtil;
48 import org.openhab.core.library.CoreItemFactory;
49 import org.openhab.core.library.items.ColorItem;
50 import org.openhab.core.library.items.ContactItem;
51 import org.openhab.core.library.items.DimmerItem;
52 import org.openhab.core.library.items.NumberItem;
53 import org.openhab.core.library.items.RollershutterItem;
54 import org.openhab.core.library.items.SwitchItem;
55 import org.openhab.core.library.types.DecimalType;
56 import org.openhab.core.library.types.OnOffType;
57 import org.openhab.core.library.types.OpenClosedType;
58 import org.openhab.core.library.types.PercentType;
59 import org.openhab.core.library.types.QuantityType;
60 import org.openhab.core.persistence.FilterCriteria;
61 import org.openhab.core.persistence.FilterCriteria.Ordering;
62 import org.openhab.core.persistence.HistoricItem;
63 import org.openhab.core.persistence.PersistenceItemInfo;
64 import org.openhab.core.persistence.PersistenceService;
65 import org.openhab.core.persistence.QueryablePersistenceService;
66 import org.openhab.core.persistence.strategy.PersistenceCronStrategy;
67 import org.openhab.core.persistence.strategy.PersistenceStrategy;
68 import org.openhab.core.types.State;
69 import org.osgi.service.component.annotations.Activate;
70 import org.osgi.service.component.annotations.Component;
71 import org.osgi.service.component.annotations.ConfigurationPolicy;
72 import org.osgi.service.component.annotations.Modified;
73 import org.osgi.service.component.annotations.Reference;
74 import org.rrd4j.ConsolFun;
75 import org.rrd4j.DsType;
76 import org.rrd4j.core.FetchData;
77 import org.rrd4j.core.FetchRequest;
78 import org.rrd4j.core.RrdDb;
79 import org.rrd4j.core.RrdDb.Builder;
80 import org.rrd4j.core.RrdDbPool;
81 import org.rrd4j.core.RrdDef;
82 import org.rrd4j.core.Sample;
83 import org.slf4j.Logger;
84 import org.slf4j.LoggerFactory;
87 * This is the implementation of the RRD4j {@link PersistenceService}. To learn
88 * more about RRD4j please visit their
89 * <a href="https://github.com/rrd4j/rrd4j">website</a>.
91 * @author Kai Kreuzer - Initial contribution
92 * @author Jan N. Klug - some improvements
93 * @author Karel Goderis - remove TimerThread dependency
96 @Component(service = { PersistenceService.class,
97 QueryablePersistenceService.class }, configurationPid = "org.openhab.rrd4j", configurationPolicy = ConfigurationPolicy.OPTIONAL)
98 public class RRD4jPersistenceService implements QueryablePersistenceService {
100 private static final String DEFAULT_OTHER = "default_other";
101 private static final String DEFAULT_NUMERIC = "default_numeric";
102 private static final String DEFAULT_QUANTIFIABLE = "default_quantifiable";
104 private static final Set<String> SUPPORTED_TYPES = Set.of(CoreItemFactory.SWITCH, CoreItemFactory.CONTACT,
105 CoreItemFactory.DIMMER, CoreItemFactory.NUMBER, CoreItemFactory.ROLLERSHUTTER, CoreItemFactory.COLOR);
107 private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(3,
108 new NamedThreadFactory("RRD4j"));
110 private final Map<String, RrdDefConfig> rrdDefs = new ConcurrentHashMap<>();
112 private static final String DATASOURCE_STATE = "state";
114 private static final Path DB_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "persistence", "rrd4j").toAbsolutePath();
116 private static final RrdDbPool DATABASE_POOL = new RrdDbPool();
118 private final Logger logger = LoggerFactory.getLogger(RRD4jPersistenceService.class);
120 private final Map<String, ScheduledFuture<?>> scheduledJobs = new HashMap<>();
122 private final ItemRegistry itemRegistry;
124 public static Path getDatabasePath(String name) {
125 return DB_FOLDER.resolve(name + ".rrd");
128 public static RrdDbPool getDatabasePool() {
129 return DATABASE_POOL;
133 public RRD4jPersistenceService(final @Reference ItemRegistry itemRegistry) {
134 this.itemRegistry = itemRegistry;
138 public String getId() {
143 public String getLabel(@Nullable Locale locale) {
148 public synchronized void store(final Item item, @Nullable final String alias) {
149 if (!isSupportedItemType(item)) {
150 logger.trace("Ignoring item '{}' since its type {} is not supported", item.getName(), item.getType());
153 final String name = alias == null ? item.getName() : alias;
155 RrdDb db = getDB(name);
160 ConsolFun function = getConsolidationFunction(db);
161 long now = System.currentTimeMillis() / 1000;
162 if (function != ConsolFun.AVERAGE) {
164 // we store the last value again, so that the value change
165 // in the database is not interpolated, but
166 // happens right at this spot
167 if (now - 1 > db.getLastUpdateTime()) {
168 // only do it if there is not already a value
169 double lastValue = db.getLastDatasourceValue(DATASOURCE_STATE);
170 if (!Double.isNaN(lastValue)) {
171 Sample sample = db.createSample();
172 sample.setTime(now - 1);
173 sample.setValue(DATASOURCE_STATE, lastValue);
175 logger.debug("Stored '{}' as value '{}' in rrd4j database (again)", name, lastValue);
178 } catch (IOException e) {
179 logger.debug("Error storing last value (again): {}", e.getMessage());
183 Sample sample = db.createSample();
188 if (item instanceof NumberItem && item.getState() instanceof QuantityType) {
189 NumberItem nItem = (NumberItem) item;
190 QuantityType<?> qState = (QuantityType<?>) item.getState();
191 Unit<? extends Quantity<?>> unit = nItem.getUnit();
193 QuantityType<?> convertedState = qState.toUnit(unit);
194 if (convertedState != null) {
195 value = convertedState.doubleValue();
198 "Failed to convert state '{}' to unit '{}'. Please check your item definition for correctness.",
202 value = qState.doubleValue();
205 DecimalType state = item.getStateAs(DecimalType.class);
207 value = state.toBigDecimal().doubleValue();
211 if (db.getDatasource(DATASOURCE_STATE).getType() == DsType.COUNTER) { // counter values must be
212 // adjusted by stepsize
213 value = value * db.getRrdDef().getStep();
215 sample.setValue(DATASOURCE_STATE, value);
217 logger.debug("Stored '{}' as value '{}' in rrd4j database", name, value);
219 } catch (IllegalArgumentException e) {
220 String message = e.getMessage();
221 if (message != null && message.contains("at least one second step is required")) {
222 // we try to store the value one second later
223 ScheduledFuture<?> job = scheduledJobs.get(name);
226 scheduledJobs.remove(name);
228 job = scheduler.schedule(() -> store(item, name), 1, TimeUnit.SECONDS);
229 scheduledJobs.put(name, job);
231 logger.warn("Could not persist '{}' to rrd4j database: {}", name, e.getMessage());
233 } catch (Exception e) {
234 logger.warn("Could not persist '{}' to rrd4j database: {}", name, e.getMessage());
238 } catch (IOException e) {
239 logger.debug("Error closing rrd4j database: {}", e.getMessage());
244 public void store(Item item) {
249 public Iterable<HistoricItem> query(FilterCriteria filter) {
250 String itemName = filter.getItemName();
252 RrdDb db = getDB(itemName);
254 logger.debug("Could not find item '{}' in rrd4j database", itemName);
261 item = itemRegistry.getItem(itemName);
262 if (item instanceof NumberItem) {
263 // we already retrieve the unit here once as it is a very costly operation,
264 // see https://github.com/openhab/openhab-addons/issues/8928
265 unit = ((NumberItem) item).getUnit();
267 } catch (ItemNotFoundException e) {
268 logger.debug("Could not find item '{}' in registry", itemName);
272 long end = filter.getEndDate() == null ? System.currentTimeMillis() / 1000
273 : filter.getEndDate().toInstant().getEpochSecond();
276 if (filter.getBeginDate() == null) {
277 // as rrd goes back for years and gets more and more
278 // inaccurate, we only support descending order
279 // and a single return value
280 // if there is no begin date is given - this case is
281 // required specifically for the historicState()
282 // query, which we want to support
283 if (filter.getOrdering() == Ordering.DESCENDING && filter.getPageSize() == 1
284 && filter.getPageNumber() == 0) {
285 if (filter.getEndDate() == null) {
286 // we are asked only for the most recent value!
287 double lastValue = db.getLastDatasourceValue(DATASOURCE_STATE);
288 if (!Double.isNaN(lastValue)) {
289 HistoricItem rrd4jItem = new RRD4jItem(itemName, mapToState(lastValue, item, unit),
290 ZonedDateTime.ofInstant(Instant.ofEpochMilli(db.getLastArchiveUpdateTime() * 1000),
291 ZoneId.systemDefault()));
292 return List.of(rrd4jItem);
300 throw new UnsupportedOperationException("rrd4j does not allow querys without a begin date, "
301 + "unless order is descending and a single value is requested");
304 start = filter.getBeginDate().toInstant().getEpochSecond();
307 FetchRequest request = db.createFetchRequest(getConsolidationFunction(db), start, end, 1);
308 FetchData result = request.fetchData();
310 List<HistoricItem> items = new ArrayList<>();
311 long ts = result.getFirstTimestamp();
312 long step = result.getRowCount() > 1 ? result.getStep() : 0;
313 for (double value : result.getValues(DATASOURCE_STATE)) {
314 if (!Double.isNaN(value) && (((ts >= start) && (ts <= end)) || (start == end))) {
315 RRD4jItem rrd4jItem = new RRD4jItem(itemName, mapToState(value, item, unit),
316 ZonedDateTime.ofInstant(Instant.ofEpochSecond(ts), ZoneId.systemDefault()));
317 items.add(rrd4jItem);
322 } catch (IOException e) {
323 logger.warn("Could not query rrd4j database for item '{}': {}", itemName, e.getMessage());
328 } catch (IOException e) {
329 logger.debug("Error closing rrd4j database: {}", e.getMessage());
335 public Set<PersistenceItemInfo> getItemInfo() {
339 protected synchronized @Nullable RrdDb getDB(String alias) {
341 Path path = getDatabasePath(alias);
343 Builder builder = RrdDb.getBuilder();
344 builder.setPool(DATABASE_POOL);
346 if (Files.exists(path)) {
347 // recreate the RrdDb instance from the file
348 builder.setPath(path.toString());
349 db = builder.build();
351 if (!Files.exists(DB_FOLDER)) {
352 Files.createDirectories(DB_FOLDER);
354 RrdDef rrdDef = getRrdDef(alias, path);
355 if (rrdDef != null) {
356 // create a new database file
357 builder.setRrdDef(rrdDef);
358 db = builder.build();
361 "Did not create rrd4j database for item '{}' since no rrd definition could be determined. This is likely due to an unsupported item type.",
365 } catch (IOException e) {
366 logger.error("Could not create rrd4j database file '{}': {}", path, e.getMessage());
367 } catch (RejectedExecutionException e) {
368 // this happens if the system is shut down
369 logger.debug("Could not create rrd4j database file '{}': {}", path, e.getMessage());
374 private @Nullable RrdDefConfig getRrdDefConfig(String itemName) {
375 RrdDefConfig useRdc = null;
376 for (Map.Entry<String, RrdDefConfig> e : rrdDefs.entrySet()) {
377 // try to find special config
378 RrdDefConfig rdc = e.getValue();
379 if (rdc.appliesTo(itemName)) {
384 if (useRdc == null) { // not defined, use defaults
386 Item item = itemRegistry.getItem(itemName);
387 if (!isSupportedItemType(item)) {
390 if (item instanceof NumberItem) {
391 NumberItem numberItem = (NumberItem) item;
392 useRdc = numberItem.getDimension() != null ? rrdDefs.get(DEFAULT_QUANTIFIABLE)
393 : rrdDefs.get(DEFAULT_NUMERIC);
395 useRdc = rrdDefs.get(DEFAULT_OTHER);
397 } catch (ItemNotFoundException e) {
398 logger.debug("Could not find item '{}' in registry", itemName);
402 logger.trace("Using rrd definition '{}' for item '{}'.", useRdc, itemName);
406 private @Nullable RrdDef getRrdDef(String itemName, Path path) {
407 RrdDef rrdDef = new RrdDef(path.toString());
408 RrdDefConfig useRdc = getRrdDefConfig(itemName);
409 if (useRdc != null) {
410 rrdDef.setStep(useRdc.step);
411 rrdDef.setStartTime(System.currentTimeMillis() / 1000 - 1);
412 rrdDef.addDatasource(DATASOURCE_STATE, useRdc.dsType, useRdc.heartbeat, useRdc.min, useRdc.max);
413 for (RrdArchiveDef rad : useRdc.archives) {
414 rrdDef.addArchive(rad.fcn, rad.xff, rad.steps, rad.rows);
422 public ConsolFun getConsolidationFunction(RrdDb db) {
424 return db.getRrdDef().getArcDefs()[0].getConsolFun();
425 } catch (IOException e) {
426 return ConsolFun.MAX;
430 @SuppressWarnings({ "unchecked", "rawtypes" })
431 private State mapToState(double value, @Nullable Item item, @Nullable Unit unit) {
432 if (item instanceof GroupItem) {
433 item = ((GroupItem) item).getBaseItem();
436 if (item instanceof SwitchItem && !(item instanceof DimmerItem)) {
437 return value == 0.0d ? OnOffType.OFF : OnOffType.ON;
438 } else if (item instanceof ContactItem) {
439 return value == 0.0d ? OpenClosedType.CLOSED : OpenClosedType.OPEN;
440 } else if (item instanceof DimmerItem || item instanceof RollershutterItem || item instanceof ColorItem) {
441 // make sure Items that need PercentTypes instead of DecimalTypes do receive the right information
442 return new PercentType((int) Math.round(value * 100));
443 } else if (item instanceof NumberItem) {
445 return new QuantityType(value, unit);
448 return new DecimalType(value);
451 private boolean isSupportedItemType(Item item) {
452 if (item instanceof GroupItem) {
453 final Item baseItem = ((GroupItem) item).getBaseItem();
454 if (baseItem != null) {
459 return SUPPORTED_TYPES.contains(ItemUtil.getMainItemType(item.getType()));
463 protected void activate(final Map<String, Object> config) {
468 protected void modified(final Map<String, Object> config) {
469 // clean existing definitions
472 // add default configurations
474 RrdDefConfig defaultNumeric = new RrdDefConfig(DEFAULT_NUMERIC);
475 // use 10 seconds as a step size for numeric values and allow a 10 minute silence between updates
476 defaultNumeric.setDef("GAUGE,600,U,U,10");
477 // define 5 different boxes:
478 // 1. granularity of 10s for the last hour
479 // 2. granularity of 1m for the last week
480 // 3. granularity of 15m for the last year
481 // 4. granularity of 1h for the last 5 years
482 // 5. granularity of 1d for the last 10 years
484 .addArchives("LAST,0.5,1,360:LAST,0.5,6,10080:LAST,0.5,90,36500:LAST,0.5,360,43800:LAST,0.5,8640,3650");
485 rrdDefs.put(DEFAULT_NUMERIC, defaultNumeric);
487 RrdDefConfig defaultQuantifiable = new RrdDefConfig(DEFAULT_QUANTIFIABLE);
488 // use 10 seconds as a step size for numeric values and allow a 10 minute silence between updates
489 defaultQuantifiable.setDef("GAUGE,600,U,U,10");
490 // define 5 different boxes:
491 // 1. granularity of 10s for the last hour
492 // 2. granularity of 1m for the last week
493 // 3. granularity of 15m for the last year
494 // 4. granularity of 1h for the last 5 years
495 // 5. granularity of 1d for the last 10 years
496 defaultQuantifiable.addArchives(
497 "AVERAGE,0.5,1,360:AVERAGE,0.5,6,10080:AVERAGE,0.5,90,36500:AVERAGE,0.5,360,43800:AVERAGE,0.5,8640,3650");
498 rrdDefs.put(DEFAULT_QUANTIFIABLE, defaultQuantifiable);
500 RrdDefConfig defaultOther = new RrdDefConfig(DEFAULT_OTHER);
501 // use 5 seconds as a step size for discrete values and allow a 1h silence between updates
502 defaultOther.setDef("GAUGE,3600,U,U,5");
503 // define 4 different boxes:
504 // 1. granularity of 5s for the last hour
505 // 2. granularity of 1m for the last week
506 // 3. granularity of 15m for the last year
507 // 4. granularity of 4h for the last 10 years
508 defaultOther.addArchives("LAST,0.5,1,720:LAST,0.5,12,10080:LAST,0.5,180,35040:LAST,0.5,2880,21900");
509 rrdDefs.put(DEFAULT_OTHER, defaultOther);
511 if (config.isEmpty()) {
512 logger.debug("using default configuration only");
516 Iterator<String> keys = config.keySet().iterator();
517 while (keys.hasNext()) {
518 String key = keys.next();
520 if ("service.pid".equals(key) || "component.name".equals(key)) {
521 // ignore service.pid and name
525 String[] subkeys = key.split("\\.");
526 if (subkeys.length != 2) {
527 logger.debug("config '{}' should have the format 'name.configkey'", key);
531 Object v = config.get(key);
532 if (v instanceof String) {
533 String value = (String) v;
534 String name = subkeys[0].toLowerCase();
535 String property = subkeys[1].toLowerCase();
537 if (value.isBlank()) {
538 logger.trace("Config is empty: {}", property);
541 logger.trace("Processing config: {} = {}", property, value);
544 RrdDefConfig rrdDef = rrdDefs.get(name);
545 if (rrdDef == null) {
546 rrdDef = new RrdDefConfig(name);
547 rrdDefs.put(name, rrdDef);
551 if ("def".equals(property)) {
552 rrdDef.setDef(value);
553 } else if ("archives".equals(property)) {
554 rrdDef.addArchives(value);
555 } else if ("items".equals(property)) {
556 rrdDef.addItems(value);
558 logger.debug("Unknown property {} : {}", property, value);
560 } catch (IllegalArgumentException e) {
561 logger.warn("Ignoring illegal configuration: {}", e.getMessage());
566 for (RrdDefConfig rrdDef : rrdDefs.values()) {
567 if (rrdDef.isValid()) {
568 logger.debug("Created {}", rrdDef);
570 logger.info("Removing invalid definition {}", rrdDef);
571 rrdDefs.remove(rrdDef.name);
576 private static class RrdArchiveDef {
577 public @Nullable ConsolFun fcn;
579 public int steps, rows;
582 public String toString() {
583 StringBuilder sb = new StringBuilder(" " + fcn);
584 sb.append(" xff = ").append(xff);
585 sb.append(" steps = ").append(steps);
586 sb.append(" rows = ").append(rows);
587 return sb.toString();
591 private class RrdDefConfig {
593 public @Nullable DsType dsType;
594 public int heartbeat, step;
595 public double min, max;
596 public List<RrdArchiveDef> archives;
597 public List<String> itemNames;
599 private boolean isInitialized;
601 public RrdDefConfig(String name) {
603 archives = new ArrayList<>();
604 itemNames = new ArrayList<>();
605 isInitialized = false;
608 public void setDef(String defString) {
609 String[] opts = defString.split(",");
610 if (opts.length != 5) { // check if correct number of parameters
611 logger.warn("invalid number of parameters {}: {}", name, defString);
615 if ("ABSOLUTE".equals(opts[0])) { // dsType
616 dsType = DsType.ABSOLUTE;
617 } else if ("COUNTER".equals(opts[0])) {
618 dsType = DsType.COUNTER;
619 } else if ("DERIVE".equals(opts[0])) {
620 dsType = DsType.DERIVE;
621 } else if ("GAUGE".equals(opts[0])) {
622 dsType = DsType.GAUGE;
624 logger.warn("{}: dsType {} not supported", name, opts[0]);
627 heartbeat = Integer.parseInt(opts[1]);
629 if ("U".equals(opts[2])) {
632 min = Double.parseDouble(opts[2]);
635 if ("U".equals(opts[3])) {
638 max = Double.parseDouble(opts[3]);
641 step = Integer.parseInt(opts[4]);
643 isInitialized = true; // successfully initialized
648 public void addArchives(String archivesString) {
649 String splitArchives[] = archivesString.split(":");
650 for (String archiveString : splitArchives) {
651 String[] opts = archiveString.split(",");
652 if (opts.length != 4) { // check if correct number of parameters
653 logger.warn("invalid number of parameters {}: {}", name, archiveString);
656 RrdArchiveDef arc = new RrdArchiveDef();
658 if ("AVERAGE".equals(opts[0])) {
659 arc.fcn = ConsolFun.AVERAGE;
660 } else if ("MIN".equals(opts[0])) {
661 arc.fcn = ConsolFun.MIN;
662 } else if ("MAX".equals(opts[0])) {
663 arc.fcn = ConsolFun.MAX;
664 } else if ("LAST".equals(opts[0])) {
665 arc.fcn = ConsolFun.LAST;
666 } else if ("FIRST".equals(opts[0])) {
667 arc.fcn = ConsolFun.FIRST;
668 } else if ("TOTAL".equals(opts[0])) {
669 arc.fcn = ConsolFun.TOTAL;
671 logger.warn("{}: consolidation function {} not supported", name, opts[0]);
673 arc.xff = Double.parseDouble(opts[1]);
674 arc.steps = Integer.parseInt(opts[2]);
675 arc.rows = Integer.parseInt(opts[3]);
680 public void addItems(String itemsString) {
681 Collections.addAll(itemNames, itemsString.split(","));
684 public boolean appliesTo(String item) {
685 return itemNames.contains(item);
688 public boolean isValid() { // a valid configuration must be initialized
689 // and contain at least one function
690 return isInitialized && !archives.isEmpty();
694 public String toString() {
695 StringBuilder sb = new StringBuilder(name);
696 sb.append(" = ").append(dsType);
697 sb.append(" heartbeat = ").append(heartbeat);
698 sb.append(" min/max = ").append(min).append("/").append(max);
699 sb.append(" step = ").append(step);
700 sb.append(" ").append(archives.size()).append(" archives(s) = [");
701 for (RrdArchiveDef arc : archives) {
702 sb.append(arc.toString());
705 sb.append(itemNames.size()).append(" items(s) = [");
706 for (String item : itemNames) {
707 sb.append(item).append(" ");
710 return sb.toString();
715 public List<PersistenceStrategy> getDefaultStrategies() {
716 return List.of(PersistenceStrategy.Globals.RESTORE, PersistenceStrategy.Globals.CHANGE,
717 new PersistenceCronStrategy("everyMinute", "0 * * * * ?"));