commit 10/31
This commit is contained in:
366
datahandling.py
366
datahandling.py
@@ -6,28 +6,124 @@ import requests
|
||||
import json
|
||||
from cell_connection import initializeUartPort, baseOperations, initializeLogs
|
||||
import logging
|
||||
from requests_toolbelt import MultipartEncoder
|
||||
import post_encode
|
||||
import base64
|
||||
import binascii
|
||||
import array
|
||||
import csv
|
||||
|
||||
# logging.basicConfig(level=logging.INFO)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
LOGGER_LEVEL=logging.DEBUG
|
||||
CONSOLE_LOGGER_LEVEL=logging.DEBUG
|
||||
|
||||
# LOGGER_LEVEL=logging.DEBUG
|
||||
# CONSOLE_LOGGER_LEVEL=logging.DEBUG
|
||||
#
|
||||
SCRIPTDIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# handler = logging.StreamHandler()
|
||||
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
|
||||
# handler.setFormatter(formatter)
|
||||
# logger.addHandler(handler)
|
||||
# logger.setLevel(logging.INFO)
|
||||
|
||||
# logger_gsm = logging.getLogger("cell_connection")
|
||||
# handler = logging.StreamHandler()
|
||||
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
|
||||
# handler.setFormatter(formatter)
|
||||
# logger_gsm.addHandler(handler)
|
||||
# logger_gsm.setLevel(logging.DEBUG)
|
||||
|
||||
# logger_1 = logging.getLogger("gsm")
|
||||
# handler = logging.StreamHandler()
|
||||
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
|
||||
# handler.setFormatter(formatter)
|
||||
# logger_1.addHandler(handler)
|
||||
# logger_1.setLevel(logging.DEBUG)
|
||||
#
|
||||
# logger_2 = logging.getLogger("intetgsm")
|
||||
# handler = logging.StreamHandler()
|
||||
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
|
||||
# handler.setFormatter(formatter)
|
||||
# logger_2.addHandler(handler)
|
||||
# logger_2.setLevel(logging.DEBUG)
|
||||
|
||||
def add_keys_if_necessary(existing_file, test_dict):
|
||||
all=None
|
||||
try:
|
||||
with open(existing_file,'r') as f:
|
||||
reader = csv.reader(f)
|
||||
header = []
|
||||
try:
|
||||
header = next(reader)
|
||||
except StopIteration as e:
|
||||
pass
|
||||
|
||||
if len(header) == 0:
|
||||
header = list(test_dict.keys())
|
||||
writer = csv.DictWriter(existing_file, test_dict.keys(), extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
|
||||
# if not header >= list(test_dict.keys()) or not header <= list(test_dict.keys()):
|
||||
if len(set (header)-set(test_dict.keys()))>0:
|
||||
existing_keys =set (header)
|
||||
data_keys = set (test_dict.keys())
|
||||
keys_to_add = data_keys - existing_keys
|
||||
all = []
|
||||
for key in keys_to_add:
|
||||
header.append(key)
|
||||
all.append(header)
|
||||
for row in reader:
|
||||
all.append(row)
|
||||
|
||||
pass
|
||||
except (OSError, IOError, TypeError) as e:
|
||||
header = list(test_dict.keys())
|
||||
f=open(existing_file, 'a+')
|
||||
writer = csv.DictWriter(f, test_dict.keys(), extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
f.close()
|
||||
|
||||
if all is not None:
|
||||
with open (existing_file, 'w') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerows(all)
|
||||
return header
|
||||
|
||||
class Datalogger_Debug ():
|
||||
def __init__(self, path=SCRIPTDIR):
|
||||
def __init__(self, text_path,
|
||||
log_path,
|
||||
photo_path):
|
||||
self.text_path=text_path
|
||||
self.log_path=os.path.join(log_path, datetime.datetime.now().strftime("Log %Y%m%d-%H%M%S.csv"))
|
||||
self.photo_path=photo_path
|
||||
pass
|
||||
|
||||
def log(self, message, type="text"):
|
||||
if type == "text":
|
||||
print ("%s - Log Message: %s"% (str(datetime.datetime.now()), message))
|
||||
elif type == "image":
|
||||
print ("%s - Log Image: %s"% (str(datetime.datetime.now()), message))
|
||||
elif type == "data":
|
||||
|
||||
|
||||
def log(self, message, message_type="text"):
|
||||
if message_type == "text":
|
||||
print ("%s - Log Message: %s"% (str(datetime.datetime.now()), message))
|
||||
elif message_type == "image":
|
||||
image = message
|
||||
if image:
|
||||
filename = datetime.datetime.now().strftime("Image %Y%m%d-%H%M%S.jpg")
|
||||
with open(os.path.join(self.photo_path, filename), 'wb') as f:
|
||||
f.write(message)
|
||||
return 'success'
|
||||
elif message_type == "data":
|
||||
message["sent"] = str(datetime.datetime.now())
|
||||
file=self.log_path
|
||||
|
||||
header = add_keys_if_necessary(file, message)
|
||||
keys = header
|
||||
|
||||
log=open(file, 'a')
|
||||
writer = csv.DictWriter(log, keys, extrasaction="ignore")
|
||||
writer.writerow(message)
|
||||
log.close()
|
||||
else:
|
||||
raise Exception
|
||||
|
||||
@@ -58,7 +154,7 @@ class Datareporter_Debug2 ():
|
||||
report_url = "http://10.0.1.4:5010/report",
|
||||
report_image_url = "http://10.0.1.4:5010/photo",
|
||||
com_port_name = "/dev/ttyAMA0",
|
||||
baud_rate = "9600"):
|
||||
baud_rate = 9600):
|
||||
self.report_url = report_url
|
||||
self.report_image_url = report_image_url
|
||||
self.com_port_name = com_port_name,
|
||||
@@ -106,126 +202,168 @@ class Datareporter_Debug3 ():
|
||||
#Debug 2 sends from cell to server
|
||||
from lib.sim900.inetgsm import SimInetGSM
|
||||
|
||||
def __init__(self, path=SCRIPTDIR,
|
||||
report_url = "http://10.0.1.4:5010/report",
|
||||
report_image_url = "http://10.0.1.4:5010/upload-file",
|
||||
com_port_name = "/dev/ttyAMA0",
|
||||
baud_rate = "9600",
|
||||
content_type = None):
|
||||
def __init__(self,
|
||||
url,
|
||||
data_path,
|
||||
image_path,
|
||||
ping_path,
|
||||
server_port,
|
||||
com_port_name=None,
|
||||
baud_rate=None,
|
||||
use_lan = False,
|
||||
path=SCRIPTDIR):
|
||||
#TODO communication
|
||||
self.report_url = report_url
|
||||
self.report_image_url = report_image_url
|
||||
self.url = url
|
||||
self.server_port=server_port
|
||||
self.image_path = image_path
|
||||
self.data_path = data_path
|
||||
self.com_port_name = com_port_name
|
||||
self.baud_rate = baud_rate
|
||||
self.content_type = content_type
|
||||
|
||||
self.ping_path = ping_path
|
||||
self.use_lan = use_lan
|
||||
self.port = initializeUartPort(portName=self.com_port_name, baudrate=self.baud_rate)
|
||||
if not use_lan:
|
||||
d = baseOperations(self.port, logger)
|
||||
|
||||
if not d is None:
|
||||
# return None
|
||||
|
||||
(self.gsm, self.imei) = d
|
||||
|
||||
self.inet = self.SimInetGSM(self.port, logger)
|
||||
|
||||
logger.info("ip = {0}".format(self.inet.ip))
|
||||
|
||||
pass
|
||||
|
||||
@property
|
||||
def status (self):
|
||||
#TODO status check
|
||||
try:
|
||||
check = json.loads(request.urlopen(self.report_url).read().decode()).get('status')
|
||||
pass
|
||||
if not check:
|
||||
return (0, "Data reporter functioning properly")
|
||||
else:
|
||||
return (1, check)
|
||||
|
||||
return (0, "Data reporter functioning properly")
|
||||
|
||||
except Exception as e:
|
||||
return (1, "Data reporter error: %s" % e)
|
||||
|
||||
def send(self, message, type="text"):
|
||||
if type == "text":
|
||||
def send(self, message, message_type):
|
||||
# logger.debug("Message.read is {}".format(message.read()))
|
||||
if message_type == "ping":
|
||||
#TODO send text
|
||||
print ("%s - Sent Message: %s"% (str(datetime.datetime.now()), message))
|
||||
elif type == "image":
|
||||
#todo send image
|
||||
# response = requests.post(self.report_image_url, files={'file': message})
|
||||
print ("%s - Sent Image: %s"% (str(datetime.datetime.now()), message))
|
||||
contentType='text/xml'
|
||||
|
||||
if self.use_lan:
|
||||
req = request.Request("{0}:{1}/{2}".format(self.url, self.server_port, self.ping_path))
|
||||
req.add_header('Content-Type', contentType)
|
||||
response = request.urlopen(req,json.dumps(message).encode())
|
||||
response = response.read()
|
||||
pass
|
||||
|
||||
elif not self.use_lan:
|
||||
logger.info("making HTTP POST request from cell")
|
||||
logger.info("attaching GPRS")
|
||||
|
||||
if not self.inet.attachGPRS("wholesale", "", "", 1):
|
||||
logger.error("error attaching GPRS")
|
||||
return False
|
||||
|
||||
if not self.inet.httpPOST(
|
||||
self.url,
|
||||
self.server_port,
|
||||
"/{}".format(self.ping_path),
|
||||
json.dumps(message),
|
||||
contentType=contentType
|
||||
):
|
||||
logger.error("error making HTTP GET post: {0}".format(self.inet.errorText))
|
||||
return False
|
||||
|
||||
response=self.inet.httpResponse
|
||||
|
||||
if response is not None:
|
||||
response = str(self.inet.httpResponse).replace("\n\r", "\n")
|
||||
else:
|
||||
response = ("empty response")
|
||||
|
||||
|
||||
elif message_type == "image":
|
||||
contentType="mulipart/form-data"
|
||||
re=message
|
||||
response=None
|
||||
m = MultipartEncoder(fields={'image': ('image', message, 'image/jpeg')})
|
||||
|
||||
|
||||
if self.use_lan:
|
||||
# req.add_header('Content-Type', contentType)
|
||||
response = requests.post("{0}:{1}/{2}".format(self.url, self.server_port, self.image_path), data=m.read(), headers={'Content-Type': m.content_type})
|
||||
# response = requests.post("{0}:{1}/{2}".format(self.url, self.server_port, self.image_path), data=m)
|
||||
# response = response.text()
|
||||
pass
|
||||
|
||||
elif not self.use_lan:
|
||||
logger.info("attaching GPRS")
|
||||
if not self.inet.attachGPRS("wholesale", "", "", 1):
|
||||
logger.error("error attaching GPRS")
|
||||
return False
|
||||
|
||||
logger.info("ip = {0}".format(self.inet.ip))
|
||||
|
||||
#making HTTP GET request
|
||||
logger.info("making HTTP POST request")
|
||||
|
||||
|
||||
if not self.inet.httpPOST(
|
||||
self.url,
|
||||
self.server_port,
|
||||
"/{}".format(self.image_path),
|
||||
m.to_string(),
|
||||
contentType=m.content_type
|
||||
):
|
||||
logger.error("error making HTTP POST: {0}".format(self.inet.errorText))
|
||||
return False
|
||||
|
||||
if self.inet.httpResponse is not None:
|
||||
response = str(self.inet.httpResponse).replace("\n\r", "\n")
|
||||
else:
|
||||
response = "empty response"
|
||||
|
||||
#adding & initializing port object
|
||||
port = initializeUartPort(portName=self.com_port_name, baudrate=self.baud_rate)
|
||||
|
||||
#initializing logger
|
||||
(formatter, logger, consoleLogger,) = initializeLogs(LOGGER_LEVEL, CONSOLE_LOGGER_LEVEL)
|
||||
|
||||
#making base operations
|
||||
d = baseOperations(port, logger)
|
||||
if d is None:
|
||||
return False
|
||||
|
||||
(gsm, imei) = d
|
||||
|
||||
inet = self.SimInetGSM(port, logger)
|
||||
|
||||
logger.info("attaching GPRS")
|
||||
if not inet.attachGPRS("wholesale", "", "", 1):
|
||||
logger.error("error attaching GPRS")
|
||||
return False
|
||||
|
||||
logger.info("ip = {0}".format(inet.ip))
|
||||
|
||||
elif message_type == "data":
|
||||
#making HTTP GET request
|
||||
logger.info("making HTTP POST request")
|
||||
contentType="application/json"
|
||||
message["sent"] = str(datetime.datetime.now())
|
||||
|
||||
if not inet.httpPOST_DATA(
|
||||
"home.ascorrea.com",
|
||||
5010,
|
||||
"/upload-file",
|
||||
# content=self.content_type,
|
||||
parameters="{0}".format(message)
|
||||
):
|
||||
logger.error("error making HTTP GET post: {0}".format(inet.errorText))
|
||||
return False
|
||||
if self.use_lan:
|
||||
req = request.Request("{0}:{1}/{2}".format(self.url, self.server_port, self.data_path))
|
||||
req.add_header('Content-Type', contentType)
|
||||
response = request.urlopen(req,json.dumps(message).encode())
|
||||
response = response.read()
|
||||
pass
|
||||
|
||||
logger.info("httpResult = {0}".format(inet.httpResult))
|
||||
if inet.httpResponse is not None:
|
||||
response = str(inet.httpResponse).replace("\n\r", "\n")
|
||||
elif not self.use_lan:
|
||||
logger.info("making HTTP POST request from cell")
|
||||
logger.info("attaching GPRS")
|
||||
|
||||
if not self.inet.attachGPRS("wholesale", "", "", 1):
|
||||
logger.error("error attaching GPRS")
|
||||
return False
|
||||
|
||||
if not self.inet.httpPOST(
|
||||
self.url,
|
||||
self.server_port,
|
||||
"/{}".format(self.data_path),
|
||||
json.dumps(message),
|
||||
contentType=contentType
|
||||
):
|
||||
logger.error("error making HTTP GET post: {0}".format(self.inet.errorText))
|
||||
return False
|
||||
|
||||
if response is not None:
|
||||
response = str(self.inet.httpResponse).replace("\n\r", "\n")
|
||||
else:
|
||||
response = ("empty response")
|
||||
|
||||
if response is not None:
|
||||
logger.info("response: \"{0}\"".format(response))
|
||||
else:
|
||||
logger.info("empty response")
|
||||
|
||||
elif type == "data":
|
||||
#adding & initializing port object
|
||||
port = initializeUartPort(portName=self.com_port_name, baudrate=self.baud_rate)
|
||||
|
||||
#initializing logger
|
||||
(formatter, logger, consoleLogger,) = initializeLogs(LOGGER_LEVEL, CONSOLE_LOGGER_LEVEL)
|
||||
|
||||
#making base operations
|
||||
d = baseOperations(port, logger)
|
||||
if d is None:
|
||||
return False
|
||||
|
||||
(gsm, imei) = d
|
||||
|
||||
inet = self.SimInetGSM(port, logger)
|
||||
|
||||
logger.info("attaching GPRS")
|
||||
if not inet.attachGPRS("internet", "", "", 1):
|
||||
logger.error("error attaching GPRS")
|
||||
return False
|
||||
|
||||
logger.info("ip = {0}".format(inet.ip))
|
||||
|
||||
#making HTTP GET request
|
||||
logger.info("making HTTP POST request")
|
||||
|
||||
|
||||
|
||||
if not inet.httpPOST(
|
||||
"home.ascorrea.com",
|
||||
5010,
|
||||
"/report-encoded",
|
||||
parse.urlencode(message)
|
||||
):
|
||||
logger.error("error making HTTP GET post: {0}".format(inet.errorText))
|
||||
return False
|
||||
|
||||
logger.info("httpResult = {0}".format(inet.httpResult))
|
||||
if inet.httpResponse is not None:
|
||||
response = str(inet.httpResponse).replace("\n\r", "\n")
|
||||
logger.info("response: \"{0}\"".format(response))
|
||||
else:
|
||||
else:
|
||||
logger.info("empty response")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user