1 contributor
#!/usr/bin/python3
# For Multithread
from threading import Thread
# For HTTP server
from http.server import BaseHTTPRequestHandler, HTTPServer
import os
from urllib.parse import urlparse
import socket
import requests
import configuration
# For MQTT and plot part
import time
import csv2geojs
import paho.mqtt.client as mqtt
# Common
import re
import json
from userio import *
MAXLRR=4
server = None
broker = None
latestHtml = ""
previousLastFcnt = 0
def htmlLatest(deveui,subID,FCnt,timeLatest,lat,lon,err,rssi,comment=None,solver=0):
global latestHtml
global previousLastFcnt
htmlOutput="<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\n"
htmlOutput+=csv2geojs.networkSurveyAddLeaflet()
htmlOutput+="<script type=\"text/javascript\">\n"
htmlOutput+="setTimeout(function(){\n"
htmlOutput+=" location = ''\n"
htmlOutput+="},1200000);\n"
htmlOutput+="</script>\n"
htmlOutput+="<body>\n"
htmlOutput+="<h2>"+server['name']+" Latest</h2>\n"
htmlOutput+="<p>\n<div id=\"home\"><button class=\"btn-green-menu\" onClick=\"window.location.href='.';\">Home</div>\n"
htmlOutput+="<div id=\"log\"><button class=\"btn-green-menu\" onClick=\"window.location.href='./log';\">Log</div>\n"
#htmlOutput+=" <div id=\"mapid\" style=\"width: 95%; max-width: 400px; height: 200px; margin: auto\"></div>\n"
htmlOutput+=" <div id=\"mapid\"></div>\n"
if 0 != lat and 0 != lon:
xmin=0.999999*lon
xmax=1.000001*lon
ymin=0.999999*lat
ymax=1.000001*lat
htmlOutput+="<script>\n"
zoomValue=19
circleOpacity=0.2
color=csv2geojs.networkSurveyColor(rssi)
if err < 3:
err=3
htmlOutput+=" var mymap = L.map('mapid').setView(["+str(lat)+", "+str(lon)+"], "+str(zoomValue)+");\n"
htmlOutput+="\n"
htmlOutput+=" L.tileLayer('https://api.mapbox.com/styles/v1/{id}/tiles/{z}/{x}/{y}?access_token="+csv2geojs.networkSurveySetMapboxApiToken()+"', {\n"
htmlOutput+=" maxZoom: 20,\n"
htmlOutput+=" attribution: 'Map data © <a href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap</a> contributors, ' +\n"
htmlOutput+=" 'Imagery © <a href=\"https://www.mapbox.com/\">Mapbox</a>',\n"
htmlOutput+=" id: 'mapbox/streets-v11',\n"
htmlOutput+=" tileSize: 512,\n"
htmlOutput+=" zoomOffset: -1\n"
htmlOutput+=" }).addTo(mymap);\n"
#DO not plot TPXLE err radius
if solver == 0:
htmlOutput+=" L.circle(["+str(lat)+","+str(lon)+"],{color: '"+color+"', fillcolor: '"+color+"', fillOpacity: "+str(circleOpacity)+", radius:"+str(err)+"}).addTo(mymap);\n"
else:
htmlOutput+=" L.circle(["+str(lat)+","+str(lon)+"],{color: '"+color+"', fillcolor: '"+color+"', fillOpacity: "+str(circleOpacity)+", radius:"+str(10)+"}).addTo(mymap);\n"
htmlOutput+="</script>\n"
htmlOutput+="<p>\n"
htmlOutput+="<div id=\"deveui\">"+deveui+"</div>\n"
htmlOutput+="<div id=\"fcnt\">"+str(FCnt)+"</div>\n"
htmlOutput+="<div id=\"time\">"+timeLatest+"</div>\n"
htmlOutput+="<div id=\"position\">"+str(lat)+"° "+str(lon)+"° "+str(err)+"m</div>\n"
htmlOutput+="<div id=\"rssi\">"+str(rssi)+"dB</div>\n"
htmlOutput+="<div id=\"delta\">Δ: "+str(FCnt-previousLastFcnt)+"</div>\n"
if comment is not None:
htmlOutput+="<div id=\"comment\">"+comment+"</div>\n"
previousLastFcnt = FCnt
htmlOutput+="</p>\n"
htmlOutput+="<p>\n<div id=\"reload\"><button class=\"btn-green\" onClick=\"window.location.reload();\">Reload</div>\n</p>\n"
htmlOutput+="</body>\n"
htmlOutput+="</html>\n"
latestHtml = htmlOutput
# The callback for when the client receives a CONNACK response from the server.
def on_connect(client, userdata, flags, rc):
if 0 == rc:
ok("MQTT Connected with result code "+str(rc))
client.subscribe("geoloc/#")
else:
error("MQTT error code "+str(rc))
# The callback for when a PUBLISH message is received from the server.
def on_message(client, userdata, msg):
json_data=str(msg.payload.decode('ASCII'))
parseTopic=msg.topic.split('/')
doProcess=True
try:
parsed=json.loads(json_data)
doProcess=True
except:
error("JSON parse failure")
doProcess=False
return
try:
# To differenciate AS message vs TPXLE message
#print("Uplink received for SubID: "+parseTopic[1]+" and DevEUI: "+parseTopic[3])
FCnt=parsed['processedFeed']['sequenceNumber']
devEUI=parseTopic[3].upper()
#print(parsed)
if "NEW" == parsed['validityState']:
posType=parsed['rawPosition']['rawPositionType']
#print(parsed)
#say("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" "+posType)
err=parsed['rawPosition']['horizontalAccuracy']
lat=parsed['rawPosition']['coordinates'][1]
lon=parsed['rawPosition']['coordinates'][0]
err=parsed['rawPosition']['horizontalAccuracy']
toto=parsed['rawPosition']['coordinates']
messageType="POSITION_MESSAGE"
# Gateway
snr=parsed['processedFeed']['processedPacket']['SNR']
rssi=parsed['processedFeed']['processedPacket']['RSSI']
lrrid=parsed['processedFeed']['processedPacket']['baseStationId']
timeP=parsed['time'].replace("Z","+00:00")
htmlLatest(devEUI,parseTopic[1],FCnt,timeP,lat,lon,err,snr,posType,solver=1)
ok("TPXLE UL ID: "+parseTopic[1]+" D: "+devEUI+" F: "+str(FCnt)+" Type: "+posType+" "+str(lat)+" "+str(lon)+" "+str(err))
pattern = '%Y-%m-%dT%H:%M:%S.%f%z'
epoch = int(time.mktime(time.strptime(parsed['time'], pattern)))
csvLine=parseTopic[1]+","+devEUI+","+timeP+","+str(epoch)+","+str(FCnt)+","+str(lat)+","+str(lon)+","+str(err)+","+lrrid+","+str(rssi)+","+str(snr)+",,,,,,,,,1"
with open(server['rawCsv'],'a+') as f:
f.write(csvLine+"\n")
f.close()
#print(csvLine)
else:
warn("TPXLE UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCntUp)+" TPXLE frame not supported")
return
except:
doProcess=True
if "uplink" in msg.topic and doProcess == True:
csvLine=""
#print("Uplink received for SubID: "+parseTopic[1]+" and DevEUI: "+parseTopic[3])
#print(parsed)
try:
pattern = '%Y-%m-%dT%H:%M:%S.%f%z'
epoch = int(time.mktime(time.strptime(parsed['DevEUI_uplink']['Time'], pattern)))
FCnt=parsed['DevEUI_uplink']['FCntUp']
csvLine+=parseTopic[1]+","+parseTopic[3]+","+parsed['DevEUI_uplink']['Time']+","+str(epoch)+","+str(FCnt)
except:
# Test if notification
try:
dlFPort=parsed['DevEUI_downlink_Sent']['FPort']
dlFCntDn=parsed['DevEUI_downlink_Sent']['FCntDn']
say("DL ID: "+parseTopic[1]+" DevEUI: "+parseTopic[3]+" F: "+str(dlFCntDn)+" Fport: "+str(dlFPort))
except:
try:
say("Notification "+parseTopic[1]+" DevEUI: "+parseTopic[3]+" "+parsed['DevEUI_notification']['Type'])
except:
#with open(server['logfile'],'a+') as f:
# f.write("------------------------------\nMQTT Error: topic: "+msg.topic+" content: "+msg.payload.decode('ASCII')+"\n------------------------------\n")
# f.close()
#warn("Uh ho")
pass
return
messageType = "UNKNOWN"
posType= "UNKNOWN"
try:
messageType = parsed['DevEUI_uplink']['payload']['messageType']
posType = parsed['DevEUI_uplink']['payload']['rawPositionType']
except:
pass
lat=0
lon=0
err=0
isPosition=True
if parsed['DevEUI_uplink']['payload'] is not None:
if parsed['DevEUI_uplink']['payload']['messageType'] == "POSITION_MESSAGE":
if posType == "GPS_TIMEOUT":
# Message is GPS timeout
warn("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType+" "+posType)
htmlLatest(parseTopic[3],parseTopic[1],FCnt,parsed['DevEUI_uplink']['Time'],0,0,0,parsed['DevEUI_uplink']['LrrSNR'],"GPS TImeout")
else:
try:
lat=parsed['DevEUI_uplink']['payload']['gpsLatitude']
lon=parsed['DevEUI_uplink']['payload']['gpsLongitude']
err=parsed['DevEUI_uplink']['payload']['horizontalAccuracy']
except:
warn("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType+" "+posType)
return
else:
isPosition=False
say("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType)
htmlLatest(parseTopic[3],parseTopic[1],FCnt,parsed['DevEUI_uplink']['Time'],lat,lon,err,parsed['DevEUI_uplink']['LrrSNR'],messageType)
return
else:
isPosition=False
warn("No decoded payload")
with open(server['logfile'],'a+') as f:
f.write("------------------------------\nMQTT Not decoded: topic: "+msg.topic+" content: "+msg.payload.decode('ASCII')+"\n------------------------------\n")
f.close()
return
csvLine+=","+str(lat)+","+str(lon)+","+str(err)
if 0 != lat and 0 != lon:
ok("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType+" "+str(lat)+" "+str(lon)+" "+str(err))
# Append GW details
#parsed['DevEUI_uplink']['Lrrs']['Lrr']
#Lrrid
#LrrRSSI
#LrrSNR
count=0
rssi=100
rssi=parsed['DevEUI_uplink']['LrrSNR']
for lrr in parsed['DevEUI_uplink']['Lrrs']['Lrr']:
count+=1
#print(lrr['Lrrid'])
csvLine+=","+lrr['Lrrid']+","+str(lrr['LrrRSSI'])+","+str(lrr['LrrSNR'])
i = 0
for i in range(count,MAXLRR):
csvLine+=",,,"
# Appending only if position is different from 0,0
if 0 != lat and 0 != lon:
with open(server['rawCsv'],'a+') as f:
f.write(csvLine+"\n")
f.close()
if True == isPosition:
htmlLatest(parseTopic[3],parseTopic[1],FCnt,parsed['DevEUI_uplink']['Time'],lat,lon,err,rssi,solver=0)
def httpFromParse(obj):
if ".css" not in obj.path and "favicon" not in obj.path and ".png" not in obj.path and ".js" not in obj.path:
if obj.headers['X-Real-IP'] is None:
say("From: "+obj.client_address[0]+" GET Received : "+obj.path)
else:
say("From: "+obj.headers['X-Real-IP']+" GET Received : "+obj.path)
def httpSubIdList():
global server
subIdList = []
with open(server['rawCsv'],'rt') as f:
lines=f.readlines()
for line in lines:
part = line.split(',')
subId=part[0]
if subId not in subIdList:
subIdList.append(subId)
f.close()
subIdList.sort()
content=""
content += "<ul class=\"subIdList\">"
for subId in subIdList:
content += "<li><a href=\"?subId="+subId+"\">"+subId+"</a></li>"
content += "</ul>"
return content
def httpDevEuiList(subId):
global server
devEuiList = []
with open(server['rawCsv'],'rt') as f:
lines=f.readlines()
for line in lines:
part = line.split(',')
if subId == part[0]:
if part[1] not in devEuiList:
devEuiList.append(part[1])
f.close()
devEuiList.sort()
content=""
content += "<ul class=\"devEuiList\">\n"
content += "<li><a href=\"?subId="+subId+"\">All</a> Last <a href=\"?subId="+subId+"&latest=10\">10</a> <a href=\"?subId="+subId+"&latest=100\">100</a> <a href=\"?subId="+subId+"&latest=1000\">1k</a> </li>\n"
for devEui in devEuiList:
content += "<li><a href=\"?subId="+subId+"&devEUI="+devEui+"\">"+devEui+"</a> Last <a href=\"?subId="+subId+"&devEUI="+devEui+"&latest=10\">10</a> <a href=\"?subId="+subId+"&devEUI="+devEui+"&latest=100\">100</a> <a href=\"?subId="+subId+"&devEUI="+devEui+"&latest=1000\">1k</a></li>\n"
content += "</ul>"
return content
def escape_ansi(line):
ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')
return ansi_escape.sub('', line)
def httpReverseLog(numLines):
global server
content=""
#print("Log file:"+server['logfile'])
cpt = 0
content+="<h2>"+server['name']+" Log</h2>\n"
content+="<p>\n<div id=\"latest\"><button class=\"btn-green-menu\" onClick=\"window.location.href='./last';\">Latest</div>\n"
content+="<div id=\"home\"><button class=\"btn-green-menu\" onClick=\"window.location.href='.';\">Home</div>\n"
content+="<div id=\"reload\"><button class=\"btn-green\" onClick=\"window.location.reload();\">Reload</div>\n</p>\n"
content+="<div class=\"loglines\">\n"
for line in reversed(open(server['logfile']).readlines()):
lineColor=line.rstrip()
lineColor =escape_ansi(lineColor)
lineColor = lineColor.replace("[OK]","<div class=\"logOK\">[OK]</div>")
lineColor = lineColor.replace("[WARN]","<div class=\"logWARN\">[WARN]</div>")
lineColor = lineColor.replace("[ERR]","<div class=\"logERROR\">[ERR]</div>")
lineColor = lineColor.replace(server['name']+" ","")
lineColor = lineColor.replace("+0000","")
if not "/log/" in lineColor and not "------------------------------" in lineColor and not "From: " in lineColor:
cpt += 1
content+=lineColor+"<br>\n"
if cpt >= numLines:
continue
content+="</div>\n"
return content
class MyServer(BaseHTTPRequestHandler):
def log_message(self, format, *args):
# To silence the default output of server (too verbose)
return
def do_GET(self):
global server
global latestHtml
rootdir = server['root']
httpFromParse(self)
# Load skeletons
page = []
data_page = ""
data_begin = ""
with open(configuration.get_pageBegin(),'r') as fStart:
data_begin += fStart.read().replace( 'CSTAPPNAME', server['name'] )
fStart.close()
data_end = ""
with open(configuration.get_pageEnd(),'r') as fEnd:
data_end += fEnd.read().replace( 'CSTAPPNAME', server['name'] ).replace( 'CSTAPPVERSION', server['version'] )
fEnd.close()
# Parse URI params
if "?" in self.path:
uriObject = urlparse(self.path)
queries = uriObject.query.split('&')
uriParams = {}
data_page = ""
for query in queries:
element=query.split("=")
uriParams[element[0]]=element[1]
latestNum = 0
try:
subId=uriParams['subId']
except:
subId=None
try:
devEUI=uriParams['devEUI']
except:
devEUI=None
try:
latestNum=uriParams['latest']
except:
latestNum=0
if "/favicon.png" in self.path or "/style.css" in self.path:
splitPath=self.path.split('/')
localFilename=splitPath[len(splitPath)-1]
#print(localFilename)
try:
f = open(rootdir + "/" + localFilename,'rb') #open requested file
self.send_response(200)
if self.path.endswith('.css'):
self.send_header('Content-type','text/css')
elif self.path.endswith('.bmp'):
self.send_header('Content-type','image/x-ms-bmp')
elif self.path.endswith('.png'):
self.send_header('Content-type','image/png')
elif self.path.endswith('.jpg'):
self.send_header('Content-type','image/jpeg')
else:
self.send_header('Content-type','text/html')
self.send_header('Server',server['name']+" v"+server['version'])
self.end_headers()
self.wfile.write(f.read())
f.close()
return
except:
with open(server['logfile'],'a+') as f:
f.write("------------------------------\nHTTP 404 "+self.path+"\n------------------------------\n")
f.close()
self.send_header('Server',server['name']+" v"+server['version'])
self.send_error(404, 'file not found')
elif self.path == "/" or self.path == "/index.html":
# Send list of SubID
data_page ="<h2>"+server['name']+"</h2>\n"
data_page += "<p>\n<div id=\"latest\"><button class=\"btn-green-menu\" onClick=\"window.location.href='./last';\">Latest</div>\n"
data_page += "<div id=\"log\"><button class=\"btn-green-menu\" onClick=\"window.location.href='./log';\">Log</div>\n</p>\n"
data_page += "<h3>subId List</h3>\n"
data_page += httpSubIdList()
page.append(data_begin)
page.append(data_page)
page.append(data_end)
content = ''.join(page)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header('Server',server['name']+" v"+server['version'])
self.end_headers()
self.wfile.write(content.encode('utf-8'))
elif self.path == "/last" or self.path == "/last/" or self.path == "/last/index.html":
page.append(data_begin)
page.append(latestHtml)
page.append(data_end)
content = ''.join(page)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header('Server',server['name']+" v"+server['version'])
self.end_headers()
self.wfile.write(content.encode('utf-8'))
elif self.path == "/log" or self.path == "/log/" or self.path == "/log/index.html":
uriObject = urlparse(self.path)
queries = uriObject.query.split('&')
uriParams = {}
try:
for query in queries:
element=query.split("=")
uriParams[element[0]]=element[1]
except:
pass
try:
numLines=uriParams['last']
except:
numLines = 50
page.append(data_begin)
data_page = httpReverseLog(numLines)
page.append(data_page)
page.append(data_end)
content = ''.join(page)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header('Server',server['name']+" v"+server['version'])
self.end_headers()
self.wfile.write(content.encode('utf-8'))
elif "/?" in self.path:
data_page += "<h2>"+server['name']+" "+subId+"</h2>\n"
data_page += "<p><div id=\"home\"><button class=\"btn-green-menu\" onClick=\"window.location.href='.';\">Home</div>\n"
data_page += "<div id=\"latest\"><button class=\"btn-green-menu\" onClick=\"window.location.href='./last';\">Latest</div>\n"
data_page += "<div id=\"log\"><button class=\"btn-green-menu\" onClick=\"window.location.href='./log';\">Log</div>\n"
if subId is not None and devEUI is None:
data_page += csv2geojs.networkSurvey(server['rawCsv'], subId, fileOutput=False, latestRecords=latestNum)
elif subId is not None and devEUI is not None:
data_page += csv2geojs.networkSurvey(server['rawCsv'], subId, devEUI, fileOutput=False, latestRecords=latestNum)
data_page += httpDevEuiList(subId)
page.append(data_begin)
page.append(data_page)
page.append(data_end)
content = ''.join(page)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header('Server',server['name']+" v"+server['version'])
self.end_headers()
self.wfile.write(content.encode('utf-8'))
else:
try:
f = open(rootdir + self.path,'rb') #open requested file
self.send_response(200)
if self.path.endswith('.css'):
self.send_header('Content-type','text/css')
elif self.path.endswith('.bmp'):
self.send_header('Content-type','image/x-ms-bmp')
elif self.path.endswith('.png'):
self.send_header('Content-type','image/png')
elif self.path.endswith('.jpg'):
self.send_header('Content-type','image/jpeg')
else:
self.send_header('Content-type','text/html')
self.send_header('Server',server['name']+" v"+server['version'])
self.end_headers()
self.wfile.write(f.read())
f.close()
return
except IOError:
with open(server['logfile'],'a+') as f:
f.write("------------------------------\nHTTP 404 "+self.path+"\n------------------------------\n")
f.close()
self.send_header('Server',server['name']+" v"+server['version'])
self.send_error(404, 'file not found')
def mqttInitRun():
global broker
broker = configuration.get_broker()
if broker is not None:
ok("MQTT configuration read successfully")
# MQTT
client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message
try:
client.username_pw_set(broker['mqttUser'], password=broker['mqttPassword'])
except:
error("MQTT Set")
try:
say("MQTT connecting "+broker['mqttUser']+"@"+broker['address']+":"+str(broker['port']))
client.connect(broker['address'], broker['port'], 60)
except:
error("MQTT connect")
client.loop_forever()
def httpInitRun():
global server
server = configuration.get_server()
if server is not None:
ok("HTTP configuration read successfully")
say("HTTP "+server['name']+" v"+server['version'])
webServer = HTTPServer((server['address'], server['port']), MyServer)
say("HTTP started http://%s:%s" % (server['address'], server['port']))
htmlLatest(str(0),str(0),0,str(0),0,0,0,0,"Waiting for uplink")
try:
webServer.serve_forever()
except KeyboardInterrupt:
pass
webServer.server_close()
say("HTTP Server stopped.")
if __name__ == "__main__":
server = configuration.get_server()
say("===============================")
say("Starting")
Thread(target = mqttInitRun).start()
Thread(target = httpInitRun).start()