#!/usr/bin/python3 # For Multithread from threading import Thread # For HTTP server from http.server import BaseHTTPRequestHandler, HTTPServer import os from urllib.parse import urlparse import socket import requests import configuration # For MQTT and plot part import time import csv2geojs import paho.mqtt.client as mqtt # Common import re import json from userio import * MAXLRR=4 server = None broker = None latestHtml = "" previousLastFcnt = 0 def htmlLatest(deveui,subID,FCnt,timeLatest,lat,lon,err,rssi,comment=None,solver=0): global latestHtml global previousLastFcnt htmlOutput="\n" htmlOutput+=csv2geojs.networkSurveyAddLeaflet() htmlOutput+="\n" htmlOutput+="\n" htmlOutput+="

"+server['name']+" Latest

\n" htmlOutput+="

\n

\n" htmlOutput+="
\n" #htmlOutput+="
\n" htmlOutput+="
\n" if 0 != lat and 0 != lon: xmin=0.999999*lon xmax=1.000001*lon ymin=0.999999*lat ymax=1.000001*lat htmlOutput+="\n" htmlOutput+="

\n" htmlOutput+="

"+deveui+"
\n" htmlOutput+="
"+str(FCnt)+"
\n" htmlOutput+="
"+timeLatest+"
\n" htmlOutput+="
"+str(lat)+"° "+str(lon)+"° "+str(err)+"m
\n" htmlOutput+="
"+str(rssi)+"dB
\n" htmlOutput+="
Δ: "+str(FCnt-previousLastFcnt)+"
\n" if comment is not None: htmlOutput+="
"+comment+"
\n" previousLastFcnt = FCnt htmlOutput+="

\n" htmlOutput+="

\n

\n

\n" htmlOutput+="\n" htmlOutput+="\n" latestHtml = htmlOutput # The callback for when the client receives a CONNACK response from the server. def on_connect(client, userdata, flags, rc): if 0 == rc: ok("MQTT Connected with result code "+str(rc)) client.subscribe("geoloc/#") else: error("MQTT error code "+str(rc)) # The callback for when a PUBLISH message is received from the server. def on_message(client, userdata, msg): json_data=str(msg.payload.decode('ASCII')) parseTopic=msg.topic.split('/') doProcess=True try: parsed=json.loads(json_data) doProcess=True except: error("JSON parse failure") doProcess=False return try: # To differenciate AS message vs TPXLE message #print("Uplink received for SubID: "+parseTopic[1]+" and DevEUI: "+parseTopic[3]) FCnt=parsed['processedFeed']['sequenceNumber'] devEUI=parseTopic[3].upper() #print(parsed) if "NEW" == parsed['validityState']: posType=parsed['rawPosition']['rawPositionType'] #print(parsed) #say("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" "+posType) err=parsed['rawPosition']['horizontalAccuracy'] lat=parsed['rawPosition']['coordinates'][1] lon=parsed['rawPosition']['coordinates'][0] err=parsed['rawPosition']['horizontalAccuracy'] toto=parsed['rawPosition']['coordinates'] messageType="POSITION_MESSAGE" # Gateway snr=parsed['processedFeed']['processedPacket']['SNR'] rssi=parsed['processedFeed']['processedPacket']['RSSI'] lrrid=parsed['processedFeed']['processedPacket']['baseStationId'] timeP=parsed['time'].replace("Z","+00:00") htmlLatest(devEUI,parseTopic[1],FCnt,timeP,lat,lon,err,snr,posType,solver=1) ok("TPXLE UL ID: "+parseTopic[1]+" D: "+devEUI+" F: "+str(FCnt)+" Type: "+posType+" "+str(lat)+" "+str(lon)+" "+str(err)) pattern = '%Y-%m-%dT%H:%M:%S.%f%z' epoch = int(time.mktime(time.strptime(parsed['time'], pattern))) csvLine=parseTopic[1]+","+devEUI+","+timeP+","+str(epoch)+","+str(FCnt)+","+str(lat)+","+str(lon)+","+str(err)+","+lrrid+","+str(rssi)+","+str(snr)+",,,,,,,,,1" with open(server['rawCsv'],'a+') as f: f.write(csvLine+"\n") f.close() #print(csvLine) else: warn("TPXLE UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCntUp)+" TPXLE frame not supported") return except: doProcess=True if "uplink" in msg.topic and doProcess == True: csvLine="" #print("Uplink received for SubID: "+parseTopic[1]+" and DevEUI: "+parseTopic[3]) #print(parsed) try: pattern = '%Y-%m-%dT%H:%M:%S.%f%z' epoch = int(time.mktime(time.strptime(parsed['DevEUI_uplink']['Time'], pattern))) FCnt=parsed['DevEUI_uplink']['FCntUp'] csvLine+=parseTopic[1]+","+parseTopic[3]+","+parsed['DevEUI_uplink']['Time']+","+str(epoch)+","+str(FCnt) except: # Test if notification try: dlFPort=parsed['DevEUI_downlink_Sent']['FPort'] dlFCntDn=parsed['DevEUI_downlink_Sent']['FCntDn'] say("DL ID: "+parseTopic[1]+" DevEUI: "+parseTopic[3]+" F: "+str(dlFCntDn)+" Fport: "+str(dlFPort)) except: try: say("Notification "+parseTopic[1]+" DevEUI: "+parseTopic[3]+" "+parsed['DevEUI_notification']['Type']) except: #with open(server['logfile'],'a+') as f: # f.write("------------------------------\nMQTT Error: topic: "+msg.topic+" content: "+msg.payload.decode('ASCII')+"\n------------------------------\n") # f.close() #warn("Uh ho") pass return messageType = "UNKNOWN" posType= "UNKNOWN" try: messageType = parsed['DevEUI_uplink']['payload']['messageType'] posType = parsed['DevEUI_uplink']['payload']['rawPositionType'] except: pass lat=0 lon=0 err=0 isPosition=True if parsed['DevEUI_uplink']['payload'] is not None: if parsed['DevEUI_uplink']['payload']['messageType'] == "POSITION_MESSAGE": if posType == "GPS_TIMEOUT": # Message is GPS timeout warn("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType+" "+posType) htmlLatest(parseTopic[3],parseTopic[1],FCnt,parsed['DevEUI_uplink']['Time'],0,0,0,parsed['DevEUI_uplink']['LrrSNR'],"GPS TImeout") else: try: lat=parsed['DevEUI_uplink']['payload']['gpsLatitude'] lon=parsed['DevEUI_uplink']['payload']['gpsLongitude'] err=parsed['DevEUI_uplink']['payload']['horizontalAccuracy'] except: warn("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType+" "+posType) return else: isPosition=False say("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType) htmlLatest(parseTopic[3],parseTopic[1],FCnt,parsed['DevEUI_uplink']['Time'],lat,lon,err,parsed['DevEUI_uplink']['LrrSNR'],messageType) return else: isPosition=False warn("No decoded payload") with open(server['logfile'],'a+') as f: f.write("------------------------------\nMQTT Not decoded: topic: "+msg.topic+" content: "+msg.payload.decode('ASCII')+"\n------------------------------\n") f.close() return csvLine+=","+str(lat)+","+str(lon)+","+str(err) if 0 != lat and 0 != lon: ok("UL ID: "+parseTopic[1]+" D: "+parseTopic[3]+" F: "+str(FCnt)+" Type: "+messageType+" "+str(lat)+" "+str(lon)+" "+str(err)) # Append GW details #parsed['DevEUI_uplink']['Lrrs']['Lrr'] #Lrrid #LrrRSSI #LrrSNR count=0 rssi=100 rssi=parsed['DevEUI_uplink']['LrrSNR'] for lrr in parsed['DevEUI_uplink']['Lrrs']['Lrr']: count+=1 #print(lrr['Lrrid']) csvLine+=","+lrr['Lrrid']+","+str(lrr['LrrRSSI'])+","+str(lrr['LrrSNR']) i = 0 for i in range(count,MAXLRR): csvLine+=",,," # Appending only if position is different from 0,0 if 0 != lat and 0 != lon: with open(server['rawCsv'],'a+') as f: f.write(csvLine+"\n") f.close() if True == isPosition: htmlLatest(parseTopic[3],parseTopic[1],FCnt,parsed['DevEUI_uplink']['Time'],lat,lon,err,rssi,solver=0) def httpFromParse(obj): if ".css" not in obj.path and "favicon" not in obj.path and ".png" not in obj.path and ".js" not in obj.path: if obj.headers['X-Real-IP'] is None: say("From: "+obj.client_address[0]+" GET Received : "+obj.path) else: say("From: "+obj.headers['X-Real-IP']+" GET Received : "+obj.path) def httpSubIdList(): global server subIdList = [] with open(server['rawCsv'],'rt') as f: lines=f.readlines() for line in lines: part = line.split(',') subId=part[0] if subId not in subIdList: subIdList.append(subId) f.close() subIdList.sort() content="" content += "" return content def httpDevEuiList(subId): global server devEuiList = [] with open(server['rawCsv'],'rt') as f: lines=f.readlines() for line in lines: part = line.split(',') if subId == part[0]: if part[1] not in devEuiList: devEuiList.append(part[1]) f.close() devEuiList.sort() content="" content += "" return content def escape_ansi(line): ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]') return ansi_escape.sub('', line) def httpReverseLog(numLines): global server content="" cpt = 0 content+="

"+server['name']+" Log

\n" content+="

\n

\n" content+="
\n" content+="
\n

\n" content+="
\n" for line in reversed(open(server['logfile']).readlines()): lineColor=line.rstrip() lineColor =escape_ansi(lineColor) lineColor = lineColor.replace("[OK]","
[OK]
") lineColor = lineColor.replace("[WARN]","
[WARN]
") lineColor = lineColor.replace("[ERR]","
[ERR]
") lineColor = lineColor.replace(server['name']+" ","") lineColor = lineColor.replace("+0000","") if not "/log/" in lineColor and not "------------------------------" in lineColor and not "From: " in lineColor: cpt += 1 content+=lineColor+"
\n" if cpt >= numLines: break #continue content+="
\n" return content class MyServer(BaseHTTPRequestHandler): def log_message(self, format, *args): # To silence the default output of server (too verbose) return def do_GET(self): global server global latestHtml rootdir = server['root'] httpFromParse(self) # Load skeletons page = [] data_page = "" data_begin = "" with open(configuration.get_pageBegin(),'r') as fStart: data_begin += fStart.read().replace( 'CSTAPPNAME', server['name'] ) fStart.close() data_end = "" with open(configuration.get_pageEnd(),'r') as fEnd: data_end += fEnd.read().replace( 'CSTAPPNAME', server['name'] ).replace( 'CSTAPPVERSION', server['version'] ) fEnd.close() # Parse URI params if "?" in self.path: uriObject = urlparse(self.path) queries = uriObject.query.split('&') uriParams = {} data_page = "" for query in queries: element=query.split("=") uriParams[element[0]]=element[1] latestNum = 0 try: subId=uriParams['subId'] except: subId=None try: devEUI=uriParams['devEUI'] except: devEUI=None try: latestNum=uriParams['latest'] except: latestNum=0 if "/favicon.png" in self.path or "/style.css" in self.path: splitPath=self.path.split('/') localFilename=splitPath[len(splitPath)-1] #print(localFilename) try: f = open(rootdir + "/" + localFilename,'rb') #open requested file self.send_response(200) if self.path.endswith('.css'): self.send_header('Content-type','text/css') elif self.path.endswith('.bmp'): self.send_header('Content-type','image/x-ms-bmp') elif self.path.endswith('.png'): self.send_header('Content-type','image/png') elif self.path.endswith('.jpg'): self.send_header('Content-type','image/jpeg') else: self.send_header('Content-type','text/html') self.send_header('Server',server['name']+" v"+server['version']) self.end_headers() self.wfile.write(f.read()) f.close() return except: with open(server['logfile'],'a+') as f: f.write("------------------------------\nHTTP 404 "+self.path+"\n------------------------------\n") f.close() self.send_header('Server',server['name']+" v"+server['version']) self.send_error(404, 'file not found') elif self.path == "/" or self.path == "/index.html": # Send list of SubID data_page ="

"+server['name']+"

\n" data_page += "

\n

\n" data_page += "
\n

\n" data_page += "

subId List

\n" data_page += httpSubIdList() page.append(data_begin) page.append(data_page) page.append(data_end) content = ''.join(page) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header('Server',server['name']+" v"+server['version']) self.end_headers() self.wfile.write(content.encode('utf-8')) elif self.path == "/last" or self.path == "/last/" or self.path == "/last/index.html": page.append(data_begin) page.append(latestHtml) page.append(data_end) content = ''.join(page) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header('Server',server['name']+" v"+server['version']) self.end_headers() self.wfile.write(content.encode('utf-8')) elif self.path == "/log" or self.path == "/log/" or self.path == "/log/index.html" or "/log/?" in self.path: uriObject = urlparse(self.path) queries = uriObject.query.split('&') uriParams = {} try: for query in queries: element=query.split("=") uriParams[element[0]]=element[1] except: pass try: numLines=int(uriParams['last']) except: numLines = 50 page.append(data_begin) data_page = httpReverseLog(numLines) page.append(data_page) page.append(data_end) content = ''.join(page) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header('Server',server['name']+" v"+server['version']) self.end_headers() self.wfile.write(content.encode('utf-8')) elif "/?" in self.path: data_page += "

"+server['name']+" "+subId+"

\n" data_page += "

\n" data_page += "
\n" data_page += "
\n" if subId is not None and devEUI is None: data_page += csv2geojs.networkSurvey(server['rawCsv'], subId, fileOutput=False, latestRecords=latestNum) elif subId is not None and devEUI is not None: data_page += csv2geojs.networkSurvey(server['rawCsv'], subId, devEUI, fileOutput=False, latestRecords=latestNum) data_page += httpDevEuiList(subId) page.append(data_begin) page.append(data_page) page.append(data_end) content = ''.join(page) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header('Server',server['name']+" v"+server['version']) self.end_headers() self.wfile.write(content.encode('utf-8')) else: try: f = open(rootdir + self.path,'rb') #open requested file self.send_response(200) if self.path.endswith('.css'): self.send_header('Content-type','text/css') elif self.path.endswith('.bmp'): self.send_header('Content-type','image/x-ms-bmp') elif self.path.endswith('.png'): self.send_header('Content-type','image/png') elif self.path.endswith('.jpg'): self.send_header('Content-type','image/jpeg') else: self.send_header('Content-type','text/html') self.send_header('Server',server['name']+" v"+server['version']) self.end_headers() self.wfile.write(f.read()) f.close() return except IOError: with open(server['logfile'],'a+') as f: f.write("------------------------------\nHTTP 404 "+self.path+"\n------------------------------\n") f.close() self.send_header('Server',server['name']+" v"+server['version']) self.send_error(404, 'file not found') def mqttInitRun(): global broker broker = configuration.get_broker() if broker is not None: ok("MQTT configuration read successfully") # MQTT client = mqtt.Client() client.on_connect = on_connect client.on_message = on_message try: client.username_pw_set(broker['mqttUser'], password=broker['mqttPassword']) except: error("MQTT Set") try: say("MQTT connecting "+broker['mqttUser']+"@"+broker['address']+":"+str(broker['port'])) client.connect(broker['address'], broker['port'], 60) except: error("MQTT connect") client.loop_forever() def httpInitRun(): global server server = configuration.get_server() if server is not None: ok("HTTP configuration read successfully") say("HTTP "+server['name']+" v"+server['version']) webServer = HTTPServer((server['address'], server['port']), MyServer) say("HTTP started http://%s:%s" % (server['address'], server['port'])) htmlLatest(str(0),str(0),0,str(0),0,0,0,0,"Waiting for uplink") try: webServer.serve_forever() except KeyboardInterrupt: pass webServer.server_close() say("HTTP Server stopped.") if __name__ == "__main__": server = configuration.get_server() say("===============================") say("Starting") Thread(target = mqttInitRun).start() Thread(target = httpInitRun).start()