projects/weather-pydash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switch to pandas and csv

Browse Source
This commit is contained in:
tony 2025-01-16 10:44:17 -07:00
parent 5d0912aa84
commit 4a7ca3b5e3
3 changed files with 152 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
data-old.py Normal file
View File

@ -0,0 +1,78 @@
import requests
import ast
from datetime import datetime
from time import sleep
import pandas as pd
import pickle
import json
import math
import numpy as np
def convert_float(value):
try:
return float(value)
except:
return None
def convert_int(value):
try:
return int(value)
except:
return None
root_dir = "/home/tonydero/projects/weather-pydash/"
sensors_url = "http://192.168.1.221/get_livedata_info"
try:
with open(root_dir+'output/saved_sensors_dict.pkl', 'rb') as f:
loaded_dict = pickle.load(f)
except:
print("Failed to open pkl")
sensors_response = requests.get(sensors_url)
sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8'))
den_dict = sensors_dict['wh25'][0]
outdoor_dict = sensors_dict['ch_aisle'][0]
lroom_dict = sensors_dict['ch_aisle'][1]
roffice_dict = sensors_dict['ch_aisle'][2]
toffice_dict = sensors_dict['ch_aisle'][3]
broom_dict = sensors_dict['ch_aisle'][4]
lightning_dict = sensors_dict['lightning'][0]
datum_dict = {}
datum_dict['den_temp'] = convert_float(den_dict['intemp'])
datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100
datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg"))
datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg"))
datum_dict['lroom_temp'] = convert_float(lroom_dict['temp'])
datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100
datum_dict['lroom_batt'] = convert_int(lroom_dict['battery'])
datum_dict['roffice_temp'] = convert_float(roffice_dict['temp'])
datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100
datum_dict['roffice_batt'] = convert_int(roffice_dict['battery'])
datum_dict['toffice_temp'] = convert_float(toffice_dict['temp'])
datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100
datum_dict['toffice_batt'] = convert_int(toffice_dict['battery'])
datum_dict['broom_temp'] = convert_float(broom_dict['temp'])
datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100
datum_dict['broom_batt'] = convert_int(broom_dict['battery'])
datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp'])
datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100
datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery'])
datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance'])
datum_dict['lghtng_date'] = lightning_dict['date']
datum_dict['lghtng_time'] = lightning_dict['timestamp']
datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count'])
datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery'])
loaded_dict[int(datetime.now().strftime("%s"))] = datum_dict
print(datetime.now())
with open(root_dir+'output/saved_sensors_dict.pkl', 'wb') as f:
pickle.dump(loaded_dict, f)
with open(root_dir+'output/saved_sensors_dict.json', 'w') as f:
json.dump(loaded_dict, f)

27
data-view.py
View File

@ -5,24 +5,37 @@ import numpy as np
import math import math
def nan2None(obj): def jsonify(obj):
if isinstance(obj, dict): if isinstance(obj, dict):
return {k:nan2None(v) for k,v in obj.items()} return {jsonify(k):jsonify(v) for k,v in obj.items()}
elif isinstance(obj, list): elif isinstance(obj, list):
return [nan2None(v) for v in obj] return [jsonify(v) for v in obj]
elif isinstance(obj, float) and math.isnan(obj): elif isinstance(obj, float) and math.isnan(obj):
return None return None
# elif isinstance(obj, int) and obj > 10:
# return datetime.fromtimestamp(obj)
elif isinstance(obj, datetime):
return str(obj)
return obj return obj
root_dir = "/home/tonydero/projects/weather-pydash/" root_dir = "/home/tonydero/projects/weather-pydash/"
sensors_dict = pd.read_pickle(root_dir+"output/saved_sensors_dict.pkl") sensors_dict = pd.read_pickle(root_dir+"output/saved_sensors_dict.pkl")
none_dict = nan2None(sensors_dict) json_dict = jsonify(sensors_dict)
print(none_dict.keys()) # print(datetime.fromtimestamp(list(none_dict.keys())[0]))
# print(type(datetime.fromtimestamp(list(none_dict.keys())[0])) == datetime)
# print(type(list(none_dict.keys())[-1]) == datetime)
# print(list(none_dict.keys())[-1].strftime("%s"))
# none_df = pd.DataFrame(none_dict) # none_df = pd.DataFrame(none_dict)
# with open(root_dir+'output/none_dict.json', 'w', encoding='utf-8') as f: with open(root_dir+'output/json_dict.json', 'w', encoding='utf-8') as f:
# json.dump(none_dict, f, ensure_ascii=False, indent=4) json.dump(json_dict, f, ensure_ascii=False, indent=4)
with open(root_dir+"output/json_dict.json", "r") as jf:
json_read_dict = json.load(jf)
int_timestamps = {int(k):v for k,v in json_read_dict.items()}
print(json_dict == int_timestamps)
# sensors_df = pd.DataFrame(sensors_dict) # sensors_df = pd.DataFrame(sensors_dict)

99
data.py
View File

@ -2,6 +2,7 @@ import requests
import ast import ast
from datetime import datetime from datetime import datetime
from time import sleep from time import sleep
import pandas as pd
import pickle import pickle
import json import json
import math import math
@ -24,53 +25,61 @@ def convert_int(value):
root_dir = "/home/tonydero/projects/weather-pydash/" root_dir = "/home/tonydero/projects/weather-pydash/"
sensors_url = "http://192.168.1.221/get_livedata_info" sensors_url = "http://192.168.1.221/get_livedata_info"
sensors_response = requests.get(sensors_url) try:
sensors_out = pd.read_csv(root_dir+"output/sensors_out.csv", index_col=0)
except Exception as e:
print(datetime.now())
print(e)
sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8')) def get_sensors(url):
sensors_response = requests.get(sensors_url)
den_dict = sensors_dict['wh25'][0] sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8'))
outdoor_dict = sensors_dict['ch_aisle'][0]
lroom_dict = sensors_dict['ch_aisle'][1] den_dict = sensors_dict['wh25'][0]
roffice_dict = sensors_dict['ch_aisle'][2] outdoor_dict = sensors_dict['ch_aisle'][0]
toffice_dict = sensors_dict['ch_aisle'][3] lroom_dict = sensors_dict['ch_aisle'][1]
broom_dict = sensors_dict['ch_aisle'][4] roffice_dict = sensors_dict['ch_aisle'][2]
lightning_dict = sensors_dict['lightning'][0] toffice_dict = sensors_dict['ch_aisle'][3]
broom_dict = sensors_dict['ch_aisle'][4]
lightning_dict = sensors_dict['lightning'][0]
datum_dict = {}
datum_dict['den_temp'] = convert_float(den_dict['intemp'])
datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100
datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg"))
datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg"))
datum_dict['lroom_temp'] = convert_float(lroom_dict['temp'])
datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100
datum_dict['lroom_batt'] = convert_int(lroom_dict['battery'])
datum_dict['roffice_temp'] = convert_float(roffice_dict['temp'])
datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100
datum_dict['roffice_batt'] = convert_int(roffice_dict['battery'])
datum_dict['toffice_temp'] = convert_float(toffice_dict['temp'])
datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100
datum_dict['toffice_batt'] = convert_int(toffice_dict['battery'])
datum_dict['broom_temp'] = convert_float(broom_dict['temp'])
datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100
datum_dict['broom_batt'] = convert_int(broom_dict['battery'])
datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp'])
datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100
datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery'])
datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance'])
datum_dict['lghtng_date'] = lightning_dict['date']
datum_dict['lghtng_time'] = lightning_dict['timestamp']
datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count'])
datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery'])
datum_df = pd.DataFrame(datum_dict, index=[datetime.now()])
return datum_df
try: try:
with open(root_dir+'output/saved_sensors_dict.pkl', 'rb') as f: reading = get_sensors(sensors_url)
sensors_dict = pickle.load(f) sensors_out_new = pd.concat([sensors_out, reading])
except:
sensors_dict = {}
datum_dict = {} sensors_out_new.to_csv(root_dir+"output/sensors_out.csv")
datum_dict['den_temp'] = convert_float(den_dict['intemp']) except Exception as e:
datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100 print(datetime.now())
datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg")) print(e)
datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg"))
datum_dict['lroom_temp'] = convert_float(lroom_dict['temp'])
datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100
datum_dict['lroom_batt'] = convert_int(lroom_dict['battery'])
datum_dict['roffice_temp'] = convert_float(roffice_dict['temp'])
datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100
datum_dict['roffice_batt'] = convert_int(roffice_dict['battery'])
datum_dict['toffice_temp'] = convert_float(toffice_dict['temp'])
datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100
datum_dict['toffice_batt'] = convert_int(toffice_dict['battery'])
datum_dict['broom_temp'] = convert_float(broom_dict['temp'])
datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100
datum_dict['broom_batt'] = convert_int(broom_dict['battery'])
datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp'])
datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100
datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery'])
datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance'])
datum_dict['lghtng_date'] = lightning_dict['date']
datum_dict['lghtng_time'] = lightning_dict['timestamp']
datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count'])
datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery'])
sensors_dict[datetime.now()] = datum_dict
with open(root_dir+'output/saved_sensors_dict.pkl', 'wb') as f:
pickle.dump(sensors_dict, f)
with open(root_dir+'output/saved_sensors_dict.json', 'w', encoding='utf-8') as f:
json.dump(sensors_dict, f, ensure_ascii=False, indent=4)