switch to pandas and csv
This commit is contained in:
parent
5d0912aa84
commit
4a7ca3b5e3
78
data-old.py
Normal file
78
data-old.py
Normal file
@ -0,0 +1,78 @@
|
||||
import requests
|
||||
import ast
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
import pandas as pd
|
||||
import pickle
|
||||
import json
|
||||
import math
|
||||
import numpy as np
|
||||
|
||||
|
||||
def convert_float(value):
|
||||
try:
|
||||
return float(value)
|
||||
except:
|
||||
return None
|
||||
|
||||
def convert_int(value):
|
||||
try:
|
||||
return int(value)
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
root_dir = "/home/tonydero/projects/weather-pydash/"
|
||||
sensors_url = "http://192.168.1.221/get_livedata_info"
|
||||
|
||||
try:
|
||||
with open(root_dir+'output/saved_sensors_dict.pkl', 'rb') as f:
|
||||
loaded_dict = pickle.load(f)
|
||||
except:
|
||||
print("Failed to open pkl")
|
||||
|
||||
sensors_response = requests.get(sensors_url)
|
||||
|
||||
sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8'))
|
||||
|
||||
den_dict = sensors_dict['wh25'][0]
|
||||
outdoor_dict = sensors_dict['ch_aisle'][0]
|
||||
lroom_dict = sensors_dict['ch_aisle'][1]
|
||||
roffice_dict = sensors_dict['ch_aisle'][2]
|
||||
toffice_dict = sensors_dict['ch_aisle'][3]
|
||||
broom_dict = sensors_dict['ch_aisle'][4]
|
||||
lightning_dict = sensors_dict['lightning'][0]
|
||||
|
||||
datum_dict = {}
|
||||
datum_dict['den_temp'] = convert_float(den_dict['intemp'])
|
||||
datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100
|
||||
datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg"))
|
||||
datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg"))
|
||||
datum_dict['lroom_temp'] = convert_float(lroom_dict['temp'])
|
||||
datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100
|
||||
datum_dict['lroom_batt'] = convert_int(lroom_dict['battery'])
|
||||
datum_dict['roffice_temp'] = convert_float(roffice_dict['temp'])
|
||||
datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100
|
||||
datum_dict['roffice_batt'] = convert_int(roffice_dict['battery'])
|
||||
datum_dict['toffice_temp'] = convert_float(toffice_dict['temp'])
|
||||
datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100
|
||||
datum_dict['toffice_batt'] = convert_int(toffice_dict['battery'])
|
||||
datum_dict['broom_temp'] = convert_float(broom_dict['temp'])
|
||||
datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100
|
||||
datum_dict['broom_batt'] = convert_int(broom_dict['battery'])
|
||||
datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp'])
|
||||
datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100
|
||||
datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery'])
|
||||
datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance'])
|
||||
datum_dict['lghtng_date'] = lightning_dict['date']
|
||||
datum_dict['lghtng_time'] = lightning_dict['timestamp']
|
||||
datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count'])
|
||||
datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery'])
|
||||
|
||||
loaded_dict[int(datetime.now().strftime("%s"))] = datum_dict
|
||||
|
||||
print(datetime.now())
|
||||
with open(root_dir+'output/saved_sensors_dict.pkl', 'wb') as f:
|
||||
pickle.dump(loaded_dict, f)
|
||||
with open(root_dir+'output/saved_sensors_dict.json', 'w') as f:
|
||||
json.dump(loaded_dict, f)
|
||||
27
data-view.py
27
data-view.py
@ -5,24 +5,37 @@ import numpy as np
|
||||
import math
|
||||
|
||||
|
||||
def nan2None(obj):
|
||||
def jsonify(obj):
|
||||
if isinstance(obj, dict):
|
||||
return {k:nan2None(v) for k,v in obj.items()}
|
||||
return {jsonify(k):jsonify(v) for k,v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [nan2None(v) for v in obj]
|
||||
return [jsonify(v) for v in obj]
|
||||
elif isinstance(obj, float) and math.isnan(obj):
|
||||
return None
|
||||
# elif isinstance(obj, int) and obj > 10:
|
||||
# return datetime.fromtimestamp(obj)
|
||||
elif isinstance(obj, datetime):
|
||||
return str(obj)
|
||||
return obj
|
||||
|
||||
root_dir = "/home/tonydero/projects/weather-pydash/"
|
||||
|
||||
sensors_dict = pd.read_pickle(root_dir+"output/saved_sensors_dict.pkl")
|
||||
none_dict = nan2None(sensors_dict)
|
||||
print(none_dict.keys())
|
||||
json_dict = jsonify(sensors_dict)
|
||||
# print(datetime.fromtimestamp(list(none_dict.keys())[0]))
|
||||
# print(type(datetime.fromtimestamp(list(none_dict.keys())[0])) == datetime)
|
||||
# print(type(list(none_dict.keys())[-1]) == datetime)
|
||||
# print(list(none_dict.keys())[-1].strftime("%s"))
|
||||
# none_df = pd.DataFrame(none_dict)
|
||||
|
||||
# with open(root_dir+'output/none_dict.json', 'w', encoding='utf-8') as f:
|
||||
# json.dump(none_dict, f, ensure_ascii=False, indent=4)
|
||||
with open(root_dir+'output/json_dict.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(json_dict, f, ensure_ascii=False, indent=4)
|
||||
with open(root_dir+"output/json_dict.json", "r") as jf:
|
||||
json_read_dict = json.load(jf)
|
||||
|
||||
int_timestamps = {int(k):v for k,v in json_read_dict.items()}
|
||||
|
||||
print(json_dict == int_timestamps)
|
||||
|
||||
# sensors_df = pd.DataFrame(sensors_dict)
|
||||
|
||||
|
||||
99
data.py
99
data.py
@ -2,6 +2,7 @@ import requests
|
||||
import ast
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
import pandas as pd
|
||||
import pickle
|
||||
import json
|
||||
import math
|
||||
@ -24,53 +25,61 @@ def convert_int(value):
|
||||
root_dir = "/home/tonydero/projects/weather-pydash/"
|
||||
sensors_url = "http://192.168.1.221/get_livedata_info"
|
||||
|
||||
sensors_response = requests.get(sensors_url)
|
||||
try:
|
||||
sensors_out = pd.read_csv(root_dir+"output/sensors_out.csv", index_col=0)
|
||||
except Exception as e:
|
||||
print(datetime.now())
|
||||
print(e)
|
||||
|
||||
sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8'))
|
||||
def get_sensors(url):
|
||||
sensors_response = requests.get(sensors_url)
|
||||
|
||||
den_dict = sensors_dict['wh25'][0]
|
||||
outdoor_dict = sensors_dict['ch_aisle'][0]
|
||||
lroom_dict = sensors_dict['ch_aisle'][1]
|
||||
roffice_dict = sensors_dict['ch_aisle'][2]
|
||||
toffice_dict = sensors_dict['ch_aisle'][3]
|
||||
broom_dict = sensors_dict['ch_aisle'][4]
|
||||
lightning_dict = sensors_dict['lightning'][0]
|
||||
sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8'))
|
||||
|
||||
den_dict = sensors_dict['wh25'][0]
|
||||
outdoor_dict = sensors_dict['ch_aisle'][0]
|
||||
lroom_dict = sensors_dict['ch_aisle'][1]
|
||||
roffice_dict = sensors_dict['ch_aisle'][2]
|
||||
toffice_dict = sensors_dict['ch_aisle'][3]
|
||||
broom_dict = sensors_dict['ch_aisle'][4]
|
||||
lightning_dict = sensors_dict['lightning'][0]
|
||||
|
||||
|
||||
datum_dict = {}
|
||||
datum_dict['den_temp'] = convert_float(den_dict['intemp'])
|
||||
datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100
|
||||
datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg"))
|
||||
datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg"))
|
||||
datum_dict['lroom_temp'] = convert_float(lroom_dict['temp'])
|
||||
datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100
|
||||
datum_dict['lroom_batt'] = convert_int(lroom_dict['battery'])
|
||||
datum_dict['roffice_temp'] = convert_float(roffice_dict['temp'])
|
||||
datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100
|
||||
datum_dict['roffice_batt'] = convert_int(roffice_dict['battery'])
|
||||
datum_dict['toffice_temp'] = convert_float(toffice_dict['temp'])
|
||||
datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100
|
||||
datum_dict['toffice_batt'] = convert_int(toffice_dict['battery'])
|
||||
datum_dict['broom_temp'] = convert_float(broom_dict['temp'])
|
||||
datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100
|
||||
datum_dict['broom_batt'] = convert_int(broom_dict['battery'])
|
||||
datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp'])
|
||||
datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100
|
||||
datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery'])
|
||||
datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance'])
|
||||
datum_dict['lghtng_date'] = lightning_dict['date']
|
||||
datum_dict['lghtng_time'] = lightning_dict['timestamp']
|
||||
datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count'])
|
||||
datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery'])
|
||||
|
||||
datum_df = pd.DataFrame(datum_dict, index=[datetime.now()])
|
||||
|
||||
return datum_df
|
||||
|
||||
try:
|
||||
with open(root_dir+'output/saved_sensors_dict.pkl', 'rb') as f:
|
||||
sensors_dict = pickle.load(f)
|
||||
except:
|
||||
sensors_dict = {}
|
||||
reading = get_sensors(sensors_url)
|
||||
sensors_out_new = pd.concat([sensors_out, reading])
|
||||
|
||||
datum_dict = {}
|
||||
datum_dict['den_temp'] = convert_float(den_dict['intemp'])
|
||||
datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100
|
||||
datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg"))
|
||||
datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg"))
|
||||
datum_dict['lroom_temp'] = convert_float(lroom_dict['temp'])
|
||||
datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100
|
||||
datum_dict['lroom_batt'] = convert_int(lroom_dict['battery'])
|
||||
datum_dict['roffice_temp'] = convert_float(roffice_dict['temp'])
|
||||
datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100
|
||||
datum_dict['roffice_batt'] = convert_int(roffice_dict['battery'])
|
||||
datum_dict['toffice_temp'] = convert_float(toffice_dict['temp'])
|
||||
datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100
|
||||
datum_dict['toffice_batt'] = convert_int(toffice_dict['battery'])
|
||||
datum_dict['broom_temp'] = convert_float(broom_dict['temp'])
|
||||
datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100
|
||||
datum_dict['broom_batt'] = convert_int(broom_dict['battery'])
|
||||
datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp'])
|
||||
datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100
|
||||
datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery'])
|
||||
datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance'])
|
||||
datum_dict['lghtng_date'] = lightning_dict['date']
|
||||
datum_dict['lghtng_time'] = lightning_dict['timestamp']
|
||||
datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count'])
|
||||
datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery'])
|
||||
|
||||
sensors_dict[datetime.now()] = datum_dict
|
||||
|
||||
with open(root_dir+'output/saved_sensors_dict.pkl', 'wb') as f:
|
||||
pickle.dump(sensors_dict, f)
|
||||
with open(root_dir+'output/saved_sensors_dict.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(sensors_dict, f, ensure_ascii=False, indent=4)
|
||||
sensors_out_new.to_csv(root_dir+"output/sensors_out.csv")
|
||||
except Exception as e:
|
||||
print(datetime.now())
|
||||
print(e)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user