-
Notifications
You must be signed in to change notification settings - Fork 0
/
SampleRatio.py
161 lines (123 loc) · 5.68 KB
/
SampleRatio.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
import requests
import json
import numpy
import matplotlib.pyplot as plt
import math
from tslearn.piecewise import SymbolicAggregateApproximation
from tslearn.utils import to_time_series
from tslearn.preprocessing import TimeSeriesScalerMeanVariance
from tslearn.preprocessing import TimeSeriesResampler
##dbname = "NOAA_water_database"
##dbname = "test_quarter"
dbname = "test3" # sin pattern
def main():
# fetch original data
#for test_quarter db
## influx_url = "http://localhost:8086/query?db=" + dbname + \
## "&epoch=ms&q=SELECT+%22degrees%22+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546329600000ms+and+time+%3C%3D+1546329900000ms"
#FOR NOAA DB
## influx_url = "http://localhost:8086/query?db=" + dbname + \
## "&epoch=ms&q=SELECT+%22degrees%22+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1439856000000ms+and+time+%3C%3D+1439992520000ms+and%28%22location%22+%3D+%27santa_monica%27%29"
# For test3
influx_url = "http://localhost:8086/query?db=" + dbname + \
"&epoch=ms&q=SELECT+%22degrees%22+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546355705400ms+and+time+%3C%3D+1548969305400ms"
r = requests.get(influx_url)
json_dict = json.loads(r.content)
data = json_dict["results"][0]["series"][0]["values"]
## print(data[0])
## print(data[1])
time_interval = data[1][0] - data[0][0] # consistant time interval
print("time interval: ", time_interval)
lst2 = [item[1] for item in data]
n_segments = len(lst2)
print("original data size", len(lst2))
alphabet_size_avg = 20
#generate sample data
sample_size = 20
## sample_url = "http://localhost:8086/query?db="+dbname+\
## "&epoch=ms&q=SELECT+sample%28%22degrees%22%2C" + str(sample_size) +\
## "%29+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546329600000ms+and+time+%3C%3D+1546329900000ms"
# test3 sample (sin pattern)
sample_url = "http://localhost:8086/query?db="+dbname+\
"&epoch=ms&q=SELECT+sample%28%22degrees%22%2C" + str(sample_size) +\
"%29+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546355705400ms+and+time+%3C%3D+1548969305400ms"
## sample_url = "http://localhost:8086/query?db=" + dbname + \
## "&epoch=ms&q=SELECT+sample%28%22degrees%22%2C" + str(sample_size) +\
## "%29+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1439856000000ms+and+time+%3C%3D+1442612520000ms+and%28%22location%22+%3D+%27santa_monica%27%29"
r2 = requests.get(sample_url)
json_dict2 = json.loads(r2.content)
sampled_data = json_dict2["results"][0]["series"][0]["values"] # [[time, value], ...]
print("sample length")
print(len(sampled_data))
sample = [item[1] for item in sampled_data] #[value,...]
# print(sample)
#fill the sample data with a linear model
start_x = data[0][0]
end_x = data[-1][0]
current_x = start_x
current_loc = 0
slope = (sampled_data[current_loc][1]-sampled_data[current_loc+1][1])\
/(sampled_data[current_loc][0] - sampled_data[current_loc+1][0])
intersection = sampled_data[current_loc][1]-slope*sampled_data[current_loc][0]
sample_fit = []
end_sample_x = sampled_data[-1][0]
while current_x <= end_sample_x:
if current_x >= sampled_data[current_loc+1][0] and current_loc+1 < len(sampled_data)-1:
current_loc+=1
slope = (sampled_data[current_loc] [1]-sampled_data[current_loc+1][1]) \
/(sampled_data[current_loc][0] - sampled_data[current_loc+1][0])
intersection = sampled_data[current_loc][1] - slope*sampled_data[current_loc][0]
sample_fit.append([current_x, slope*current_x+intersection])
current_x += time_interval #1000ms
#chop the original data to match the linear fit sample data.
chopped_data = []
for item in data:
if item[0]>= sample_fit[0][0] and item[0] <= sample_fit[-1][0]:
chopped_data.append(item)
print("len")
print(len(sample_fit),len(chopped_data))
chopped_lst2 = [item[1] for item in chopped_data]
chopped_len = len(chopped_lst2)
#build a sax model for chopped original data
sax = SymbolicAggregateApproximation(chopped_len,alphabet_size_avg)
scalar = TimeSeriesScalerMeanVariance(mu=0., std=1.)
sdb = scalar.fit_transform(chopped_lst2)
sax_data = sax.transform(sdb)
s3 = sax.fit_transform(sax_data)
#build a sax model for linear-fit sampled data
sample_fit_extract = [item[1] for item in sample_fit]
fit_sample_data = scalar.fit_transform(sample_fit_extract)
sax_sample_data = sax.transform(fit_sample_data)
s4 = sax.fit_transform(sax_sample_data)
#compute the distance between to dataset to calculate the similarity
print("distance")
dist = sax.distance_sax(s3[0], s4[0])
print(dist)
print("normalized distance")
print(dist/chopped_len)
#plot the three dataset
plot(sample_fit, sampled_data, lst2)
def plot(sample_fit, sampled_data, lst2):
plt.figure()
#plot the linear fit sample data
plt.subplot(2,2,1)
x = [item[0] for item in sample_fit]
y = [item[1] for item in sample_fit]
plt.plot(x,y,'bo-')
#to temporary fix the edge issue
#plt.ylim(top = 82, bottom = 58)
plt.title("sample data (linear fill)")
#plot the sample data
plt.subplot(2,2,2)
x = [item[0] for item in sampled_data]
y = [item[1] for item in sampled_data]
plt.plot(x,y,'bo-')
plt.title("sample data")
#plot the original data
plt.subplot(2,2,3)
plt.plot(lst2,'bo-')
plt.title("original dataset")
plt.tight_layout()
plt.show()
if __name__ == "__main__":
main()