Skip to content

Commit

Permalink
RRD4j: fix default step seconds; improve post-processing
Browse files Browse the repository at this point in the history 
- default step seconds had been accidentially left at 1 second; should be 5
- update datafile migration
- improve data post-processing for uneven split
  • Loading branch information
@sfeilmeier
sfeilmeier committed Mar 23, 2021
1 parent e0753f4 commit 831c04d
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 44 deletions.
28 changes: 11 additions & 17 deletions io.openems.edge.timedata.rrd4j/src/io/openems/edge/timedata/rrd4j/Rrd4jTimedataImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public class Rrd4jTimedataImpl extends AbstractOpenemsComponent
implements Rrd4jTimedata, Timedata, OpenemsComponent, EventHandler {

protected static final String DEFAULT_DATASOURCE_NAME = "value";
protected static final int DEFAULT_STEP_SECONDS = 60;
protected static final int DEFAULT_STEP_SECONDS = 300;
protected static final int DEFAULT_HEARTBEAT_SECONDS = DEFAULT_STEP_SECONDS;

private static final String RRD4J_PATH = "rrd4j";
Expand Down Expand Up @@ -221,22 +221,16 @@ protected static double[] postProcessData(FetchRequest request, int resolution)

} else if (step > resolution) {
// Split each entry to multiple values
if (step % resolution != 0) {
throw new IllegalArgumentException(
"RRD4j Step [" + step + "] is not dividable by requested resolution [" + resolution + "]");
}
int split = (int) (step / resolution);
for (int i = 1; i < input.length; i++) {
for (int j = 0; j < split; j++) {
if ((i - 1) * split + j < result.length) {
result[(i - 1) * split + j] = input[i];
}
}
long resultTimestamp = 0;
for (int i = 0, inputIndex = 0; i < result.length; i++) {
inputIndex = Math.min(input.length - 1, (int) (resultTimestamp / step));
resultTimestamp += resolution;
result[i] = input[inputIndex];
}

} else {
// Data already matches resolution
for (int i = 1; i < input.length; i++) {
for (int i = 1; i < result.length + 1 && i < input.length; i++) {
result[i - 1] = input[i];
}
}
Expand Down Expand Up @@ -409,7 +403,7 @@ private synchronized RrdDb createNewDb(ChannelAddress channelAddress, Unit chann
RrdDef rrdDef = new RrdDef(//
this.getDbFile(channelAddress).toURI(), //
startTime, // Start-Time
DEFAULT_STEP_SECONDS // Step in [s], default: 60 = 1 minute
DEFAULT_STEP_SECONDS // Step in [s], default: 300 = 5 minutes
);
rrdDef.addDatasource(//
new DsDef(DEFAULT_DATASOURCE_NAME, //
Expand Down Expand Up @@ -545,9 +539,9 @@ private ChannelDef getDsDefForChannel(Unit channelUnit) {
*/
private RrdDb updateRrdDbToLatestDefinition(RrdDb oldDb, ChannelAddress channelAddress, Unit channelUnit)
throws IOException {
if (oldDb.getArcCount() > 2) {
if (oldDb.getArcCount() > 2 || oldDb.getRrdDef().getStep() == 60) {
/*
* This is OpenEMS-RRD4j Definition v1; migrate to v2
* This is an old OpenEMS-RRD4j Definition -> migrate to latest version
*/
// Read data of last month
long lastTimestamp = oldDb.getLastUpdateTime();
Expand Down Expand Up @@ -576,7 +570,7 @@ private RrdDb updateRrdDbToLatestDefinition(RrdDb oldDb, ChannelAddress channelA
}

this.logInfo(this.log,
"Migrate RRD4j Database [" + channelAddress.toString() + "] to OpenEMS Definition v2");
"Migrate RRD4j Database [" + channelAddress.toString() + "] to latest OpenEMS Definition");
return newDb;

} else {
Expand Down
89 changes: 62 additions & 27 deletions ...penems.edge.timedata.rrd4j/test/io/openems/edge/timedata/rrd4j/Rrd4jTimedataImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,24 +27,23 @@ private static void addSample(RrdDb database, Instant instant, double value) thr
sample.update();
}

private static RrdDb createRrdDb(int oneMinute, int fiveMinutes) throws IOException, URISyntaxException {
final RrdDef rrdDef = new RrdDef("empty-path", START.getEpochSecond(), Rrd4jTimedataImpl.DEFAULT_STEP_SECONDS);
private static RrdDb createRrdDb(int step, int fiveMinutes, int oneHour) throws IOException, URISyntaxException {
final RrdDef rrdDef = new RrdDef("empty-path", START.getEpochSecond() - 1, step);
rrdDef.addDatasource(//
new DsDef(Rrd4jTimedataImpl.DEFAULT_DATASOURCE_NAME, //
DsType.GAUGE, //
Rrd4jTimedataImpl.DEFAULT_HEARTBEAT_SECONDS, // Heartbeat in [s], default 60 = 1 minute
Rrd4jTimedataImpl.DEFAULT_HEARTBEAT_SECONDS, // Heartbeat in [s], default 300 = 5 minutes
Double.NaN, Double.NaN));
// detailed recordings
rrdDef.addArchive(ConsolFun.AVERAGE, 0.5, 1, oneMinute); // 1 step (1 minute), 1440 rows (1 day)
rrdDef.addArchive(ConsolFun.AVERAGE, 0.5, 5, fiveMinutes); // 5 steps (5 minutes), 2880 rows (10 days)
// hourly values for a very long time
rrdDef.addArchive(ConsolFun.AVERAGE, 0.5, 60, 87_600); // 60 steps (1 hour), 87600 rows (10 years)
rrdDef.addArchive(ConsolFun.AVERAGE, 0.5, 1, fiveMinutes); // 1 step (5 minutes), 8928 rows (31 days)
rrdDef.addArchive(ConsolFun.AVERAGE, 0.5, 12, oneHour); // 12 steps (60 minutes), 8016 rows (334 days)

final RrdDb database = RrdDb.getBuilder() //
.setBackendFactory(new RrdMemoryBackendFactory()) // in memory
.setRrdDef(rrdDef) //
.build();

for (int i = 1; i <= 120; i++) {
for (int i = 0; i <= 60 /* minutes */ * 4 /* hours */; i++) {
addSample(database, START.plus(i, ChronoUnit.MINUTES), i);
}
return database;
Expand All @@ -58,18 +57,20 @@ private static RrdDb createRrdDb(int oneMinute, int fiveMinutes) throws IOExcept
*/
@Test
public void testMerge() throws IOException, URISyntaxException {
int resolution = 300; // 5 minutes
int resolution = 900; // 15 minutes

RrdDb database = createRrdDb(1000, 2000);
FetchRequest request = database.createFetchRequest(ConsolFun.AVERAGE, START.getEpochSecond(),
RrdDb database = createRrdDb(300, 100, 100);
FetchRequest request = database.createFetchRequest(//
ConsolFun.AVERAGE, //
START.getEpochSecond(), //
START.plus(3, ChronoUnit.HOURS).getEpochSecond());
double[] result = Rrd4jTimedataImpl.postProcessData(request, resolution);
database.close();

assertEquals(36, result.length); // 3 hours * 12 entries/per hour (5 minutes) = 36
assertEquals(3.0, result[0], 0.001);
assertEquals(8.0, result[1], 0.001);
assertEquals(13.0, result[2], 0.001);
assertEquals(12, result.length); // 3 hours * 4 entries/per hour (15 minutes) = 12
assertEquals(8.0, result[0], 0.1);
assertEquals(23.0, result[1], 0.1);
assertEquals(38.0, result[2], 0.1);
}

/**
Expand All @@ -82,16 +83,18 @@ public void testMerge() throws IOException, URISyntaxException {
public void testExact() throws IOException, URISyntaxException {
int resolution = 300; // 5 minutes

RrdDb database = createRrdDb(10, 200);
FetchRequest request = database.createFetchRequest(ConsolFun.AVERAGE, START.getEpochSecond(),
RrdDb database = createRrdDb(300, 100, 100);
FetchRequest request = database.createFetchRequest(//
ConsolFun.AVERAGE, //
START.getEpochSecond(), //
START.plus(3, ChronoUnit.HOURS).getEpochSecond());
double[] result = Rrd4jTimedataImpl.postProcessData(request, resolution);
database.close();

assertEquals(36, result.length); // 3 hours * 12 entries/per hour (5 minutes) = 36
assertEquals(3.0, result[0], 0.001);
assertEquals(8.0, result[1], 0.001);
assertEquals(13.0, result[2], 0.001);
assertEquals(3.0, result[0], 0.1);
assertEquals(8.0, result[1], 0.1);
assertEquals(13.0, result[2], 0.1);
}

/**
Expand All @@ -102,18 +105,50 @@ public void testExact() throws IOException, URISyntaxException {
*/
@Test
public void testSplit() throws IOException, URISyntaxException {
int resolution = 300; // 5 minutes
int resolution = 60; // 1 minute

RrdDb database = createRrdDb(300, 100, 100);
FetchRequest request = database.createFetchRequest(//
ConsolFun.AVERAGE, //
START.getEpochSecond(), //
START.plus(3, ChronoUnit.HOURS).getEpochSecond());
double[] result = Rrd4jTimedataImpl.postProcessData(request, resolution);
database.close();

assertEquals(180, result.length); // 3 hours * 60 entries/per hour (1 minute) = 180
for (int i = 0; i < 5; i++) {
assertEquals(0., result[i], 0.1);
}
for (int i = 5; i < 10; i++) {
assertEquals(3., result[i], 0.1);
}
for (int i = 10; i < 15; i++) {
assertEquals(8., result[i], 0.1);
}
}

/**
* Test RRD4j step bigger than resolution, but resolution not divisible.
*
* @throws IOException on error
* @throws URISyntaxException on error
*/
@Test
public void testSplitUneven() throws IOException, URISyntaxException {
int resolution = 300; // 1 minute

RrdDb database = createRrdDb(10, 20);
FetchRequest request = database.createFetchRequest(ConsolFun.AVERAGE, START.getEpochSecond(),
RrdDb database = createRrdDb(720, 100, 100);
FetchRequest request = database.createFetchRequest(//
ConsolFun.AVERAGE, //
START.getEpochSecond(), //
START.plus(3, ChronoUnit.HOURS).getEpochSecond());
double[] result = Rrd4jTimedataImpl.postProcessData(request, resolution);
database.close();

assertEquals(36, result.length); // 3 hours * 12 entries/per hour (5 minutes) = 36
assertEquals(30.5, result[0], 0.001);
assertEquals(30.5, result[1], 0.001);
assertEquals(30.5, result[2], 0.001);
assertEquals(90.5, result[12], 0.001);
assertEquals(Double.NaN, result[0], 0.1);
assertEquals(6.5, result[3], 0.1);
assertEquals(6.5, result[4], 0.1);
assertEquals(18.5, result[5], 0.1);
}
}

0 comments on commit 831c04d

Please sign in to comment.