asc/spaceballoon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

commit 11/12

Browse Source
This commit is contained in:
Anthony Correa
2015-11-12 23:12:03 -06:00
parent 44bf6916a3
commit 1b569da791
9 changed files with 600 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

445
datahandling.py
View File

@@ -7,48 +7,12 @@ import json
from cell_connection import initializeUartPort, baseOperations, initializeLogs
import logging
from requests_toolbelt import MultipartEncoder
import post_encode
import base64
import binascii
import array
import csv
import base64
# logging.basicConfig(level=logging.INFO)
# LOGGER_LEVEL=logging.DEBUG
# CONSOLE_LOGGER_LEVEL=logging.DEBUG
#
SCRIPTDIR = os.path.dirname(os.path.abspath(__file__))
logger = logging.getLogger(__name__)
# handler = logging.StreamHandler()
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
# handler.setFormatter(formatter)
# logger.addHandler(handler)
# logger.setLevel(logging.INFO)
# logger_gsm = logging.getLogger("cell_connection")
# handler = logging.StreamHandler()
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
# handler.setFormatter(formatter)
# logger_gsm.addHandler(handler)
# logger_gsm.setLevel(logging.DEBUG)
# logger_1 = logging.getLogger("gsm")
# handler = logging.StreamHandler()
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
# handler.setFormatter(formatter)
# logger_1.addHandler(handler)
# logger_1.setLevel(logging.DEBUG)
#
# logger_2 = logging.getLogger("intetgsm")
# handler = logging.StreamHandler()
# formatter = logging.Formatter('[%(asctime)-15s] %(name)-5s %(levelname)-8s %(message)s')
# handler.setFormatter(formatter)
# logger_2.addHandler(handler)
# logger_2.setLevel(logging.DEBUG)
def add_keys_if_necessary(existing_file, test_dict):
all=None
@@ -63,7 +27,7 @@ def add_keys_if_necessary(existing_file, test_dict):
if len(header) == 0:
header = list(test_dict.keys())
writer = csv.DictWriter(existing_file, test_dict.keys(), extrasaction="ignore")
writer = csv.DictWriter(existing_file, test_dict.keys(), extrasaction='ignore')
writer.writeheader()
# if not header >= list(test_dict.keys()) or not header <= list(test_dict.keys()):
@@ -82,7 +46,7 @@ def add_keys_if_necessary(existing_file, test_dict):
except (OSError, IOError, TypeError) as e:
header = list(test_dict.keys())
f=open(existing_file, 'a+')
writer = csv.DictWriter(f, test_dict.keys(), extrasaction="ignore")
writer = csv.DictWriter(f, test_dict.keys(), extrasaction='ignore')
writer.writeheader()
f.close()
@@ -92,117 +56,59 @@ def add_keys_if_necessary(existing_file, test_dict):
writer.writerows(all)
return header
class Datalogger_Debug ():
def __init__(self, text_path,
class Datalogger():
def __init__(self,
missiontime,
text_path,
log_path,
photo_path):
self._mt = missiontime
self.text_path=text_path
self.log_path=os.path.join(log_path, datetime.datetime.now().strftime("Log %Y%m%d-%H%M%S.csv"))
self.log_path=os.path.join(log_path, 'MISSION {}.csv'.format(self._mt.name))
self.photo_path=photo_path
pass
def log(self, message, message_type="text"):
if message_type == "text":
print ("%s - Log Message: %s"% (str(datetime.datetime.now()), message))
elif message_type == "image":
image = message
if image:
filename = datetime.datetime.now().strftime("Image %Y%m%d-%H%M%S.jpg")
with open(os.path.join(self.photo_path, filename), 'wb') as f:
f.write(message)
def log(self, r):
self._record = r._get_dict()
# logger.info('Recording {}'.format(self._record))
for k in self._record.keys():
if k is 'b':
if self._record[k] is not list:
l = [self._record[k]]
else:
l = self._record[k]
for item in l:
logger.debug("item: {}".format(self._record))
item['at']=self._mt.to_absolutetime(item['mt'])
item['mid']=self._mt.name
header = add_keys_if_necessary(self.log_path, item)
keys = header
log=open(self.log_path, 'a')
writer = csv.DictWriter(log, keys, extrasaction='ignore')
writer.writerow(item)
log.close()
elif k is 'i':
if self._record[k] is tuple:
l = [self._record[k]]
else:
l = self._record[k]
for item in l:
# Form is ('name', file object, 'type')
logger.info('item is {}'.format(type(item)))
filename = item[0]
file = base64.b64decode(bytes(item[1],'ascii'))
with open(os.path.join(self.photo_path, filename), 'wb') as f:
f.write(file)
# file.save(os.path.join(self.photo_path, filename))
return 'success'
elif message_type == "data":
message["sent"] = str(datetime.datetime.now())
file=self.log_path
header = add_keys_if_necessary(file, message)
keys = header
log=open(file, 'a')
writer = csv.DictWriter(log, keys, extrasaction="ignore")
writer.writerow(message)
log.close()
else:
raise Exception
class Datareporter_Debug ():
def __init__(self, path=SCRIPTDIR):
#TODO communication
pass
@property
def status (self):
return (0, "Data reporter functioning properly")
def send(self, message, type="text"):
if type == "text":
#TODO send text
print ("%s - Sent Message: %s"% (str(datetime.datetime.now()), message))
elif type == "image":
#todo send image
print ("%s - Sent Image: %s"% (str(datetime.datetime.now()), message))
else:
#todo error handling
raise Exception
class Datareporter_Debug2 ():
#Debug 2 sends to server
def __init__(self, path=SCRIPTDIR,
report_url = "http://10.0.1.4:5010/report",
report_image_url = "http://10.0.1.4:5010/photo",
com_port_name = "/dev/ttyAMA0",
baud_rate = 9600):
self.report_url = report_url
self.report_image_url = report_image_url
self.com_port_name = com_port_name,
self.baud_rate = baud_rate
pass
@property
def status (self):
#TODO status check
try:
check = json.loads(request.urlopen(self.report_url).read().decode()).get('status')
pass
if not check:
return (0, "Data reporter functioning properly")
else:
return (1, check)
except Exception as e:
return (1, "Data reporter error: %s" % e)
def send(self, message, type="text"):
try:
if type == "text":
#TODO send text
print ("%s - Sent Message: %s"% (str(datetime.datetime.now()), message))
elif type == "image":
#todo send image
response = requests.post(self.report_url, files={'file': message})
print ("%s - Sent Image: %s"% (str(datetime.datetime.now()), message))
elif type == "data":
#add date to message
message['sent']=str(datetime.datetime.now())
req = request.Request(self.report_image_url)
req.add_header('Content-Type', 'application/json')
response = request.urlopen(req,json.dumps(message).encode())
the_page = response.read()
return 0
except Exception as e:
#todo error handling
pass
class Datareporter_Debug3 ():
class Datareporter():
#Debug 2 sends from cell to server
from lib.sim900.inetgsm import SimInetGSM
def __init__(self,
missiontime,
url,
data_path,
image_path,
@@ -213,6 +119,7 @@ class Datareporter_Debug3 ():
use_lan = False,
path=SCRIPTDIR):
#TODO communication
self.mt = missiontime
self.url = url
self.server_port=server_port
self.image_path = image_path
@@ -221,149 +128,161 @@ class Datareporter_Debug3 ():
self.baud_rate = baud_rate
self.ping_path = ping_path
self.use_lan = use_lan
self.port = initializeUartPort(portName=self.com_port_name, baudrate=self.baud_rate)
self._transpondence=None
if not use_lan:
self.port = initializeUartPort(portName=self.com_port_name, baudrate=self.baud_rate)
d = baseOperations(self.port, logger)
if not d is None:
# return None
(self.gsm, self.imei) = d
self.inet = self.SimInetGSM(self.port, logger)
self.inet = self.SimInetGSM(self.port, logger)
logger.info("ip = {0}".format(self.inet.ip))
logger.info('ip = {0}'.format(self.inet.ip))
pass
logger.debug('attaching GPRS')
if not self.inet.attachGPRS('wholesale', '', '', 1):
logger.error('error attaching GPRS')
return False
#register mission number to server
intiate = Record(str(self.mt.timezero),'zt')
# intiate.add(self.mt.mid,'mid')
self.send(intiate, _intiating_report=True)
@property
def status (self):
#TODO status check
try:
return (0, "Data reporter functioning properly")
return (0, 'Data reporter functioning properly')
except Exception as e:
return (1, "Data reporter error: %s" % e)
return (1, 'Data reporter error: %s' % e)
def send(self, message, message_type):
# logger.debug("Message.read is {}".format(message.read()))
if message_type == "ping":
#TODO send text
contentType='text/xml'
def _send_data(self, message, _intiating_report=False):
response=None
m = MultipartEncoder(fields=message)
if self.use_lan:
req = request.Request("{0}:{1}/{2}".format(self.url, self.server_port, self.ping_path))
req.add_header('Content-Type', contentType)
response = request.urlopen(req,json.dumps(message).encode())
response = response.read()
pass
elif not self.use_lan:
logger.info("making HTTP POST request from cell")
logger.info("attaching GPRS")
if not self.inet.attachGPRS("wholesale", "", "", 1):
logger.error("error attaching GPRS")
return False
if not self.inet.httpPOST(
self.url,
self.server_port,
"/{}".format(self.ping_path),
json.dumps(message),
contentType=contentType
):
logger.error("error making HTTP GET post: {0}".format(self.inet.errorText))
return False
response=self.inet.httpResponse
if response is not None:
response = str(self.inet.httpResponse).replace("\n\r", "\n")
else:
response = ("empty response")
elif message_type == "image":
contentType="mulipart/form-data"
re=message
response=None
m = MultipartEncoder(fields={'image': ('image', message, 'image/jpeg')})
if self.use_lan:
# req.add_header('Content-Type', contentType)
response = requests.post("{0}:{1}/{2}".format(self.url, self.server_port, self.image_path), data=m.read(), headers={'Content-Type': m.content_type})
# response = requests.post("{0}:{1}/{2}".format(self.url, self.server_port, self.image_path), data=m)
# response = response.text()
pass
elif not self.use_lan:
logger.info("attaching GPRS")
if not self.inet.attachGPRS("wholesale", "", "", 1):
logger.error("error attaching GPRS")
return False
logger.info("ip = {0}".format(self.inet.ip))
#making HTTP GET request
logger.info("making HTTP POST request")
if not self.inet.httpPOST(
self.url,
self.server_port,
"/{}".format(self.image_path),
m.to_string(),
contentType=m.content_type
):
logger.error("error making HTTP POST: {0}".format(self.inet.errorText))
return False
if self.inet.httpResponse is not None:
response = str(self.inet.httpResponse).replace("\n\r", "\n")
else:
response = "empty response"
elif message_type == "data":
#making HTTP GET request
contentType="application/json"
message["sent"] = str(datetime.datetime.now())
if self.use_lan:
req = request.Request("{0}:{1}/{2}".format(self.url, self.server_port, self.data_path))
req.add_header('Content-Type', contentType)
response = request.urlopen(req,json.dumps(message).encode())
response = response.read()
pass
elif not self.use_lan:
logger.info("making HTTP POST request from cell")
logger.info("attaching GPRS")
if not self.inet.attachGPRS("wholesale", "", "", 1):
logger.error("error attaching GPRS")
return False
if not self.inet.httpPOST(
self.url,
self.server_port,
"/{}".format(self.data_path),
json.dumps(message),
contentType=contentType
):
logger.error("error making HTTP GET post: {0}".format(self.inet.errorText))
return False
if response is not None:
response = str(self.inet.httpResponse).replace("\n\r", "\n")
else:
response = ("empty response")
if response is not None:
logger.info("response: \"{0}\"".format(response))
if _intiating_report:
path = '{0}'.format(self.image_path)
else:
logger.info("empty response")
path = '{0}/{1}'.format(self.image_path,self.mt.mid)
if self.use_lan:
logger.info ('Sending transpondence using LAN')
response = requests.post('{0}:{1}/{2}'.format(self.url, self.server_port, path), data=m.read(), headers={'Content-Type': m.content_type})
pass
elif not self.use_lan:
logger.info ('Sending transpondence using modem')
# logger.debug('attaching GPRS')
# if not self.inet.attachGPRS('wholesale', '', '', 1):
# logger.error('error attaching GPRS')
# return False
logger.debug('ip = {0}'.format(self.inet.ip))
logger.debug('making HTTP POST request')
if not self.inet.httpPOST(
self.url,
self.server_port,
"/{}".format(path),
m.to_string(),
contentType=m.content_type
):
logger.error('error making HTTP POST: {0}'.format(self.inet.errorText))
return False
if self.inet.httpResponse is not None:
response = str(self.inet.httpResponse).replace('\n\r', '\n')
else:
response = 'empty response'
if _intiating_report:
if type (response) is str:
mid = response
else:
mid = response.text
self.mt.set_mid(mid)
return response
def send(self, t, _intiating_report=False):
self._transpondence = t._get_dict()
self._transpondence['mt'] = self.mt.now()
# print ('Send transpondence {}'.format(self._transpondence))
for k in self._transpondence.keys():
# if self._transpondence[k] is list or self._transpondence[k] is dict or self._transpondence[k] is float:
if type(self._transpondence) is dict:
self._transpondence[k] = json.dumps(self._transpondence[k])
r = self._send_data(message=self._transpondence, _intiating_report=_intiating_report)
#On error, do not clear transpondence
if not r:
self._transpondence = self._transpondence
return None
class Record():
def __init__(self,first_item=None, first_item_type=None):
# if self._transpondence is None:
self._transpondence={}
if first_item and first_item_type:
self.add(first_item, first_item_type)
def add(self, data, info_type='data'):
if type(data) is not list:
if self._transpondence.get(info_type) is not None:
if type(self._transpondence.get(info_type)) is list:
self._transpondence[info_type].append(data)
else:
self._transpondence[info_type] = [self._transpondence[info_type]]
self._transpondence[info_type].append(data)
else:
self._transpondence[info_type]=data
else:
if self._transpondence.get(info_type) is not None:
if type(self._transpondence.get(info_type)) is list:
self._transpondence[info_type] + data
else:
self._transpondence[info_type] = [self._transpondence[info_type]]
self._transpondence[info_type] + data
else:
self._transpondence[info_type]=data
def _get_dict(self):
return self._transpondence
# def send(self):
# # self._transpondence['mid'] = m.name
# # self._transpondence['mt'] = m.now()
# # print ('Send transpondence {}'.format(self._transpondence))
#
# for k in self._transpondence.keys():
# if self._transpondence[k] is not str: #basically, check if image
# self._transpondence[k] = json.dumps(self._transpondence[k])
#
# r = self._send(message=self._transpondence, message_type='transpondence')
#
# #On error, do not clear transpondence
# if not r:
# self._transpondence = None