diff --git a/data-old.py b/data-old.py new file mode 100644 index 0000000..f58ffbd --- /dev/null +++ b/data-old.py @@ -0,0 +1,78 @@ +import requests +import ast +from datetime import datetime +from time import sleep +import pandas as pd +import pickle +import json +import math +import numpy as np + + +def convert_float(value): + try: + return float(value) + except: + return None + +def convert_int(value): + try: + return int(value) + except: + return None + + +root_dir = "/home/tonydero/projects/weather-pydash/" +sensors_url = "http://192.168.1.221/get_livedata_info" + +try: + with open(root_dir+'output/saved_sensors_dict.pkl', 'rb') as f: + loaded_dict = pickle.load(f) +except: + print("Failed to open pkl") + +sensors_response = requests.get(sensors_url) + +sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8')) + +den_dict = sensors_dict['wh25'][0] +outdoor_dict = sensors_dict['ch_aisle'][0] +lroom_dict = sensors_dict['ch_aisle'][1] +roffice_dict = sensors_dict['ch_aisle'][2] +toffice_dict = sensors_dict['ch_aisle'][3] +broom_dict = sensors_dict['ch_aisle'][4] +lightning_dict = sensors_dict['lightning'][0] + +datum_dict = {} +datum_dict['den_temp'] = convert_float(den_dict['intemp']) +datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100 +datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg")) +datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg")) +datum_dict['lroom_temp'] = convert_float(lroom_dict['temp']) +datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100 +datum_dict['lroom_batt'] = convert_int(lroom_dict['battery']) +datum_dict['roffice_temp'] = convert_float(roffice_dict['temp']) +datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100 +datum_dict['roffice_batt'] = convert_int(roffice_dict['battery']) +datum_dict['toffice_temp'] = convert_float(toffice_dict['temp']) +datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100 +datum_dict['toffice_batt'] = convert_int(toffice_dict['battery']) +datum_dict['broom_temp'] = convert_float(broom_dict['temp']) +datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100 +datum_dict['broom_batt'] = convert_int(broom_dict['battery']) +datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp']) +datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100 +datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery']) +datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance']) +datum_dict['lghtng_date'] = lightning_dict['date'] +datum_dict['lghtng_time'] = lightning_dict['timestamp'] +datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count']) +datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery']) + +loaded_dict[int(datetime.now().strftime("%s"))] = datum_dict + +print(datetime.now()) +with open(root_dir+'output/saved_sensors_dict.pkl', 'wb') as f: + pickle.dump(loaded_dict, f) +with open(root_dir+'output/saved_sensors_dict.json', 'w') as f: + json.dump(loaded_dict, f) diff --git a/data-view.py b/data-view.py index f3aac71..4573e39 100644 --- a/data-view.py +++ b/data-view.py @@ -5,24 +5,37 @@ import numpy as np import math -def nan2None(obj): +def jsonify(obj): if isinstance(obj, dict): - return {k:nan2None(v) for k,v in obj.items()} + return {jsonify(k):jsonify(v) for k,v in obj.items()} elif isinstance(obj, list): - return [nan2None(v) for v in obj] + return [jsonify(v) for v in obj] elif isinstance(obj, float) and math.isnan(obj): return None + # elif isinstance(obj, int) and obj > 10: + # return datetime.fromtimestamp(obj) + elif isinstance(obj, datetime): + return str(obj) return obj root_dir = "/home/tonydero/projects/weather-pydash/" sensors_dict = pd.read_pickle(root_dir+"output/saved_sensors_dict.pkl") -none_dict = nan2None(sensors_dict) -print(none_dict.keys()) +json_dict = jsonify(sensors_dict) +# print(datetime.fromtimestamp(list(none_dict.keys())[0])) +# print(type(datetime.fromtimestamp(list(none_dict.keys())[0])) == datetime) +# print(type(list(none_dict.keys())[-1]) == datetime) +# print(list(none_dict.keys())[-1].strftime("%s")) # none_df = pd.DataFrame(none_dict) -# with open(root_dir+'output/none_dict.json', 'w', encoding='utf-8') as f: -# json.dump(none_dict, f, ensure_ascii=False, indent=4) +with open(root_dir+'output/json_dict.json', 'w', encoding='utf-8') as f: + json.dump(json_dict, f, ensure_ascii=False, indent=4) +with open(root_dir+"output/json_dict.json", "r") as jf: + json_read_dict = json.load(jf) + +int_timestamps = {int(k):v for k,v in json_read_dict.items()} + +print(json_dict == int_timestamps) # sensors_df = pd.DataFrame(sensors_dict) diff --git a/data.py b/data.py index 4e60389..6274ec4 100644 --- a/data.py +++ b/data.py @@ -2,6 +2,7 @@ import requests import ast from datetime import datetime from time import sleep +import pandas as pd import pickle import json import math @@ -24,53 +25,61 @@ def convert_int(value): root_dir = "/home/tonydero/projects/weather-pydash/" sensors_url = "http://192.168.1.221/get_livedata_info" -sensors_response = requests.get(sensors_url) +try: + sensors_out = pd.read_csv(root_dir+"output/sensors_out.csv", index_col=0) +except Exception as e: + print(datetime.now()) + print(e) -sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8')) +def get_sensors(url): + sensors_response = requests.get(sensors_url) -den_dict = sensors_dict['wh25'][0] -outdoor_dict = sensors_dict['ch_aisle'][0] -lroom_dict = sensors_dict['ch_aisle'][1] -roffice_dict = sensors_dict['ch_aisle'][2] -toffice_dict = sensors_dict['ch_aisle'][3] -broom_dict = sensors_dict['ch_aisle'][4] -lightning_dict = sensors_dict['lightning'][0] + sensors_dict = ast.literal_eval(sensors_response.content.decode('utf-8')) + + den_dict = sensors_dict['wh25'][0] + outdoor_dict = sensors_dict['ch_aisle'][0] + lroom_dict = sensors_dict['ch_aisle'][1] + roffice_dict = sensors_dict['ch_aisle'][2] + toffice_dict = sensors_dict['ch_aisle'][3] + broom_dict = sensors_dict['ch_aisle'][4] + lightning_dict = sensors_dict['lightning'][0] + + + datum_dict = {} + datum_dict['den_temp'] = convert_float(den_dict['intemp']) + datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100 + datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg")) + datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg")) + datum_dict['lroom_temp'] = convert_float(lroom_dict['temp']) + datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100 + datum_dict['lroom_batt'] = convert_int(lroom_dict['battery']) + datum_dict['roffice_temp'] = convert_float(roffice_dict['temp']) + datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100 + datum_dict['roffice_batt'] = convert_int(roffice_dict['battery']) + datum_dict['toffice_temp'] = convert_float(toffice_dict['temp']) + datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100 + datum_dict['toffice_batt'] = convert_int(toffice_dict['battery']) + datum_dict['broom_temp'] = convert_float(broom_dict['temp']) + datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100 + datum_dict['broom_batt'] = convert_int(broom_dict['battery']) + datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp']) + datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100 + datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery']) + datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance']) + datum_dict['lghtng_date'] = lightning_dict['date'] + datum_dict['lghtng_time'] = lightning_dict['timestamp'] + datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count']) + datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery']) + + datum_df = pd.DataFrame(datum_dict, index=[datetime.now()]) + + return datum_df try: - with open(root_dir+'output/saved_sensors_dict.pkl', 'rb') as f: - sensors_dict = pickle.load(f) -except: - sensors_dict = {} + reading = get_sensors(sensors_url) + sensors_out_new = pd.concat([sensors_out, reading]) -datum_dict = {} -datum_dict['den_temp'] = convert_float(den_dict['intemp']) -datum_dict['den_humi'] = convert_float(den_dict['inhumi'].strip("%"))/100 -datum_dict['den_pabs'] = convert_float(den_dict['abs'].strip(" inHg")) -datum_dict['den_prel'] = convert_float(den_dict['rel'].strip(" inHg")) -datum_dict['lroom_temp'] = convert_float(lroom_dict['temp']) -datum_dict['lroom_humi'] = convert_float(lroom_dict['humidity'].strip("%"))/100 -datum_dict['lroom_batt'] = convert_int(lroom_dict['battery']) -datum_dict['roffice_temp'] = convert_float(roffice_dict['temp']) -datum_dict['roffice_humi'] = convert_float(roffice_dict['humidity'].strip("%"))/100 -datum_dict['roffice_batt'] = convert_int(roffice_dict['battery']) -datum_dict['toffice_temp'] = convert_float(toffice_dict['temp']) -datum_dict['toffice_humi'] = convert_float(toffice_dict['humidity'].strip("%"))/100 -datum_dict['toffice_batt'] = convert_int(toffice_dict['battery']) -datum_dict['broom_temp'] = convert_float(broom_dict['temp']) -datum_dict['broom_humi'] = convert_float(broom_dict['humidity'].strip("%"))/100 -datum_dict['broom_batt'] = convert_int(broom_dict['battery']) -datum_dict['outdoor_temp'] = convert_float(outdoor_dict['temp']) -datum_dict['outdoor_humi'] = convert_float(outdoor_dict['humidity'].strip("%"))/100 -datum_dict['outdoor_batt'] = convert_int(outdoor_dict['battery']) -datum_dict['lghtng_dist'] = convert_float(lightning_dict['distance']) -datum_dict['lghtng_date'] = lightning_dict['date'] -datum_dict['lghtng_time'] = lightning_dict['timestamp'] -datum_dict['lghtng_scnt'] = convert_float(lightning_dict['count']) -datum_dict['lghtng_batt'] = convert_int(lightning_dict['battery']) - -sensors_dict[datetime.now()] = datum_dict - -with open(root_dir+'output/saved_sensors_dict.pkl', 'wb') as f: - pickle.dump(sensors_dict, f) -with open(root_dir+'output/saved_sensors_dict.json', 'w', encoding='utf-8') as f: - json.dump(sensors_dict, f, ensure_ascii=False, indent=4) + sensors_out_new.to_csv(root_dir+"output/sensors_out.csv") +except Exception as e: + print(datetime.now()) + print(e)