Skip to content

D
df-aggregator
Commit 5b9c96dc by Oleksandr Barabash
Options

init commit. Removed update on receivers as it's going to be obsolete

parent 5eccee51
Showing with 1064 additions and 494 deletions
.gitignore
......@@ -14,3 +14,7 @@ static/node_modules
static/cesium-kml-czml-editor
package-lock.json
package.json
*.pyc
**.pyc
.venv
.idea
df-aggregator.py
#!/usr/bin/env python3
""" DF-Agg """
# df-aggregator, networked radio direction finding software.
# Copyright (C) 2020 Corey Koval
#
......@@ -15,6 +14,9 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import sys
import uuid
import vincenty as v
import numpy as np
......@@ -26,24 +28,35 @@ import signal
import json
from colorsys import hsv_to_rgb
from optparse import OptionParser
from os import system, name, kill, getpid
from lxml import etree
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler, minmax_scale
from geojson import MultiPoint, Feature, FeatureCollection
from czml3 import Packet, Document, Preamble
from czml3.properties import Position, Polyline, PolylineMaterial, PolylineOutlineMaterial, PolylineDashMaterial, Color, Material
from czml3.properties import (
Position, Polyline, PolylineMaterial, PolylineOutlineMaterial,
PolylineDashMaterial, Color, Material
)
from multiprocessing import Process, Queue
from bottle import route, run, request, get, put, response, redirect, template, static_file
from bottle.ext.websocket import GeventWebSocketServer, websocket
from bottle import (route, run, request, get, put, response,
redirect, template, static_file)
from sys import version_info
if (version_info.major != 3 or version_info.minor < 6):
print("Looks like you're running python version " +
str(version_info.major) + "." +
str(version_info.minor) + ", which is no longer supported.")
print("Your python version is out of date, please update to 3.6 or newer.")
quit()
from utils.ether_service.ether_service import EtherService
from utils.log import init_logging
logger = logging.getLogger(__name__)
if version_info.major != 3 or version_info.minor < 6:
logger.error("Looks like you're running python version "
f"{version_info.major}.{version_info.minor} "
"which is no longer supported.")
logger.error("Your python version is out of date, "
"please update to 3.6 or newer.")
sys.exit(1)
DBSCAN_Q = Queue()
......@@ -55,38 +68,66 @@ d = 40000 # draw distance of LOBs in meters
heading_d = 20000
max_age = 5000
receivers = []
cached_receivers = []
shared_dict = dict(response=None)
###############################################
# Stores settings realted to intersect capture
# and post-processing.
###############################################
# noinspection PyPep8Naming,SpellCheckingInspection
class math_settings:
""" Stores settings related to intersect capture and post-processing. """
def __init__(self, eps, min_samp, conf, power):
self.eps = eps
self.min_samp = min_samp
self.min_conf = conf
self.min_power = power
rx_busy = False
rx_busy = True
receiving = True
plotintersects = False
################################################
# Stores all variables pertaining to a reveiver.
# Also updates receiver variable upon request.
################################################
# noinspection PyPep8Naming
class receiver:
def __init__(self, station_url):
""" Stores all variables pertaining to a reveiver.
Also updates receiver variable upon request."""
latitude = 0.0
longitude = 0.0
heading = 0.0
raw_doa = 0.0
doa = 0.0
frequency = 0.0
power = 0.0
confidence = 0
doa_time = 0
isMobile = False
isSingle = False
previous_doa_time = 0
last_processed_at = 0
d_2_last_intersection = [d]
def __init__(self, station_url, station_name=None):
self.station_url = station_url
self.station_id = station_name or f"Unknown-{uuid.uuid4()}"
self.isAuto = True
self.isActive = True
self.flipped = False
self.inverted = True
self.update(first_run=True)
# self.update(first_run=True) # Don't do this! Never, ever!
# init default values
self.latitude = 0.0
self.longitude = 0.0
self.heading = 0.0
self.raw_doa = 0.0
self.doa = 0.0
self.frequency = 0.0
self.power = 0.0
self.confidence = 0
self.doa_time = 0
# Updates receiver from the remote URL
def update(self, first_run=False):
""" update """
try:
xml_contents = etree.parse(self.station_url)
xml_station_id = xml_contents.find('STATION_ID')
......@@ -117,8 +158,6 @@ class receiver:
self.power = float(xml_power.text)
xml_conf = xml_contents.find('CONF')
self.confidence = int(xml_conf.text)
except KeyboardInterrupt:
finish()
except Exception as ex:
# TODO(s1z): This is a bullshit handler!
# Has to be changed to appropriate actions depend on
......@@ -146,46 +185,40 @@ class receiver:
self.confidence = 0
self.doa_time = 0
self.isActive = False
print(ex)
print(
f"Problem connecting to {self.station_url}, receiver deactivated. Reactivate in WebUI.")
# raise IOError
exc_info = (type(ex), ex, ex.__traceback__)
logger.error("Error occurred", exc_info=exc_info)
logger.error(f"Problem connecting to {self.station_url}, "
f"receiver deactivated. Reactivate in WebUI.")
# Returns receivers properties as a dict, useful for passing data to the WebUI
def receiver_dict(self):
return ({'station_id': self.station_id, 'station_url': self.station_url,
'latitude': self.latitude, 'longitude': self.longitude, 'heading': self.heading,
'doa': self.doa, 'frequency': self.frequency, 'power': self.power,
'confidence': self.confidence, 'doa_time': self.doa_time, 'mobile': self.isMobile,
'active': self.isActive, 'auto': self.isAuto, 'inverted': self.inverted,
'single': self.isSingle})
""" Returns receivers properties as a dict,
useful for passing data to the WebUI"""
return {'station_id': self.station_id,
'station_url': self.station_url,
'latitude': self.latitude,
'longitude': self.longitude,
'heading': self.heading,
'doa': self.doa,
'frequency': self.frequency,
'power': self.power,
'confidence': self.confidence,
'doa_time': self.doa_time,
'mobile': self.isMobile,
'active': self.isActive,
'auto': self.isAuto,
'inverted': self.inverted,
'single': self.isSingle}
def lob_length(self):
""" lob_length """
if self.d_2_last_intersection:
return round(max(self.d_2_last_intersection)) + 200
else:
return d
latitude = 0.0
longitude = 0.0
heading = 0.0
raw_doa = 0.0
doa = 0.0
frequency = 0.0
power = 0.0
confidence = 0
doa_time = 0
isMobile = False
isSingle = False
previous_doa_time = 0
last_processed_at = 0
d_2_last_intersection = [d]
###############################################
# Converts Lat/Lon to polar coordinates
###############################################
def plot_polar(lat_a, lon_a, lat_a2, lon_a2):
""" Converts Lat/Lon to polar coordinates """
# Convert points in great circle 1, degrees to radians
p1_lat1_rad = math.radians(lat_a)
p1_long1_rad = math.radians(lon_a)
......@@ -206,7 +239,8 @@ def plot_polar(lat_a, lon_a, lat_a2, lon_a2):
#####################################################
# Find line of intersection between two great circles
#####################################################
def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=100000):
def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b,
max_distance=100000):
# plot another point on the lob
# v.direct(lat_a, lon_a, doa_a, d)
# returns (lat_a2, lon_a2)
......@@ -223,11 +257,12 @@ def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=10000
# Find line of intersection between two planes
L = np.cross(N1, N2)
# Find two intersection points
X1 = L / np.sqrt(L[0]**2 + L[1]**2 + L[2]**2)
X1 = L / np.sqrt(L[0] ** 2 + L[1] ** 2 + L[2] ** 2)
X2 = -X1
def mag(q):
return np.sqrt(np.vdot(q, q))
dist1 = mag(X1 - plane_a[0])
dist2 = mag(X2 - plane_a[0])
# return the (lon_lat pair of the closer intersection)
......@@ -243,33 +278,29 @@ def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=10000
if abs(check_bearing - doa_a) < 5:
km = v.inverse([lat_a, lon_a], [i_lat, i_long])
if km[0] < max_distance:
return (i_lat, i_long)
return i_lat, i_long
else:
return None
#######################################################################
# We start this in it's own process do it doesn't eat all of your RAM.
# This becomes noticable at over 10k intersections.
#######################################################################
def do_dbscan(X, epsilon, minsamp):
def do_dbscan(x, epsilon, min_samp):
""" We start this in it's own process do it doesn't eat all of your RAM.
This becomes noticable at over 10k intersections."""
DBSCAN_WAIT_Q.put(True)
db = DBSCAN(eps=epsilon, min_samples=minsamp).fit(X)
db = DBSCAN(eps=epsilon, min_samples=min_samp).fit(x)
DBSCAN_Q.put(db.labels_)
if not DBSCAN_WAIT_Q.empty():
DBSCAN_WAIT_Q.get()
####################################
# Autocalculate the best eps value.
####################################
def autoeps_calc(X):
def autoeps_calc(xs):
""" Auto calculate the best eps value. """
# only use a sample of the data to speed up calculation.
X = X[:min(2000, len(X)):2]
xs = xs[:min(2000, len(xs)):2]
min_distances = []
for x in X:
for x in xs:
distances = []
for y in X:
for y in xs:
# calculate euclidian distance
distance = math.sqrt(sum([(a - b) ** 2 for a, b in zip(x, y)]))
if distance > 0:
......@@ -292,11 +323,10 @@ def autoeps_calc(X):
return 0
###############################################
# Computes DBSCAN Alorithm is applicable,
# finds the mean of a cluster of intersections.
###############################################
# noinspection PyShadowingNames
def process_data(database_name, epsilon, min_samp):
""" Computes DBSCAN Alorithm is applicable,
finds the mean of a cluster of intersections."""
n_std = 3.0
intersect_list = []
likely_location = []
......@@ -311,7 +341,7 @@ def process_data(database_name, epsilon, min_samp):
aoi_list.append(-1)
for aoi in aoi_list:
print(f"Checking AOI {aoi}.")
logging.debug(f"Checking AOI {aoi}.")
curs.execute('''SELECT longitude, latitude, time FROM intersects
WHERE aoi_id=? ORDER BY confidence DESC LIMIT 25000''', [aoi])
intersect_array = np.array(curs.fetchall())
......@@ -330,7 +360,7 @@ def process_data(database_name, epsilon, min_samp):
if epsilon == "auto":
epsilon = autoeps_calc(X)
print(f"min_samp: {min_samp}, eps: {epsilon}")
logger.debug(f"min_samp: {min_samp}, eps: {epsilon}")
else:
try:
epsilon = float(epsilon)
......@@ -339,36 +369,37 @@ def process_data(database_name, epsilon, min_samp):
# size_x = sys.getsizeof(X)/1024
# print(f"The dataset is {size_x} kilobytes")
print(f"Computing Clusters from {n_points} intersections.")
logger.info(f"Computing Clusters "
f"from {n_points} intersections.")
while not DBSCAN_WAIT_Q.empty():
print("Waiting for my turn...")
logger.debug("Waiting for my turn...")
time.sleep(1)
starttime = time.time()
db = Process(target=do_dbscan, args=(X, epsilon, min_samp))
db.daemon = True
db.start()
# noinspection PyBroadException
try:
labels = DBSCAN_Q.get(timeout=10)
db.join()
except:
print("DBSCAN took took long, terminated.")
except Exception:
logger.warning("DBSCAN took took long, terminated.")
if not DBSCAN_WAIT_Q.empty():
DBSCAN_WAIT_Q.get()
db.terminate()
return likely_location, intersect_list, ellipsedata
stoptime = time.time()
print(
f"DBSCAN took {stoptime - starttime} seconds to compute the clusters.")
logger.info(f"DBSCAN took {stoptime - starttime}"
f"seconds to compute the clusters.")
intersect_array = np.column_stack((intersect_array, labels))
# Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
n_noise_ = list(labels).count(-1)
clear(debugging)
print('Number of clusters: %d' % n_clusters_)
print('Outliers Removed: %d' % n_noise_)
logger.debug('Number of clusters: %d' % n_clusters_)
logger.debug('Outliers Removed: %d' % n_noise_)
for x in range(n_clusters_):
cluster = np.array([]).reshape(0, 3)
......@@ -376,34 +407,39 @@ def process_data(database_name, epsilon, min_samp):
if intersect_array[y][-1] == x:
cluster = np.concatenate(
(cluster, [intersect_array[y][0:-1]]), axis=0)
# weighted_location.append(np.average(cluster[:,0:2], weights=cluster[:,2], axis=0).tolist())
# weighted_location.append(np.average(cluster[:,0:2],
# weights=cluster[:,2],
# axis=0).tolist())
clustermean = np.mean(cluster[:, 0:2], axis=0)
likely_location.append(clustermean.tolist())
cov = np.cov(cluster[:, 0], cluster[:, 1])
a = cov[0, 0]
b = cov[0, 1]
c = cov[1, 1]
if (a == 0.0 or b == 0.0 or c == 0.0):
if a == 0.0 or b == 0.0 or c == 0.0:
if debugging:
print(f"A: {a} B: {b} C: {c}")
print("Unable to resolve ellipse.")
logger.debug(f"A: {a} B: {b} C: {c}")
logger.debug("Unable to resolve ellipse.")
break
lam1 = a + c / 2 + np.sqrt((a - c / 2)**2 + b**2)
lam1 = a + c / 2 + np.sqrt((a - c / 2) ** 2 + b ** 2)
# lam2 = a+c/2 - np.sqrt((a-c/2)**2 + b**2)
# print([lam1, lam2, a, c])
pearson = b / np.sqrt(a * c)
if (1 + pearson < 0.0 or 1 - pearson < 0.0):
if 1 + pearson < 0.0 or 1 - pearson < 0.0:
if debugging:
print(f"Pearson Value: {pearson}")
print("Unable to resolve ellipse.")
logger.debug(f"Pearson Value: {pearson}")
logger.debug("Unable to resolve ellipse.")
break
print(f"A: {a} B: {b} C: {c} pearson: {pearson}")
logger.debug(f"A: {a} B: {b} C: {c} pearson: {pearson}")
ell_radius_x = np.sqrt(1 + pearson) * np.sqrt(a) * n_std
ell_radius_y = np.sqrt(1 - pearson) * np.sqrt(c) * n_std
axis_x = v.inverse(clustermean.tolist()[
::-1], (ell_radius_x + clustermean[1], clustermean[0]))[0]
::-1], (ell_radius_x + clustermean[1],
clustermean[0]))[0]
axis_y = v.inverse(clustermean.tolist()[
::-1], (clustermean[1], ell_radius_y + clustermean[0]))[0]
::-1], (clustermean[1],
ell_radius_y + clustermean[0]))[
0]
if b == 0 and a >= c:
rotation = 0
elif b == 0 and a < c:
......@@ -415,7 +451,7 @@ def process_data(database_name, epsilon, min_samp):
[axis_x, axis_y, rotation, *clustermean.tolist()])
for x in likely_location:
print(x[::-1])
logger.debug(x[::-1])
for x in intersect_array:
try:
......@@ -425,16 +461,14 @@ def process_data(database_name, epsilon, min_samp):
intersect_list.append(x.tolist())
else:
print(f"No Intersections in AOI {aoi}.")
logger.debug(f"No Intersections in AOI {aoi}.")
conn.close()
return likely_location, intersect_list, ellipsedata
#######################################################################
# Checks interesections stored in the database against a lat/lon/radius
# and removes items inside exclusion areas.
#######################################################################
def purge_database(type, lat, lon, radius):
""" Checks interesections stored in the database against a lat/lon/radius
and removes items inside exclusion areas."""
conn = sqlite3.connect(database_name)
c = conn.cursor()
c.execute("SELECT latitude, longitude, id FROM intersects")
......@@ -453,7 +487,7 @@ def purge_database(type, lat, lon, radius):
DATABASE_EDIT_Q.put((command, delete_these, False))
# DATABASE_RETURN.get(timeout=1)
DATABASE_EDIT_Q.put(("done", None, False))
print(f"I purged {purge_count} intersects.")
logger.info(f"I purged {purge_count} intersects.")
###############################################
......@@ -525,7 +559,9 @@ def run_aoi_rules():
DATABASE_EDIT_Q.put(("done", None, False))
stoptime = time.time()
print(f"Purged {purged} intersections and sorted {sorted} intersections into {n_aoi} AOIs in {stoptime - starttime} seconds.")
logger.info(f"Purged {purged} intersections and sorted {sorted}"
f"intersections into {n_aoi} AOIs"
f"in {stoptime - starttime} seconds.")
return "OK"
......@@ -538,25 +574,27 @@ def write_geojson(best_point, all_the_points):
"marker-color": "#00FF00"}
if all_the_points is not None:
all_the_points = Feature(
properties=all_pt_style, geometry=MultiPoint(tuple(all_the_points)))
properties=all_pt_style,
geometry=MultiPoint(tuple(all_the_points)))
with open(geofile, "w") as file1:
if best_point is not None:
reversed_best_point = []
for x in best_point:
reversed_best_point.append(x)
best_point = Feature(properties=best_pt_style, geometry=MultiPoint(
tuple(reversed_best_point)))
best_point = Feature(properties=best_pt_style,
geometry=MultiPoint(
tuple(reversed_best_point)))
file1.write(str(FeatureCollection(
[best_point, all_the_points])))
else:
file1.write(str(FeatureCollection([all_the_points])))
print(f"Wrote file {geofile}")
logger.debug(f"Wrote file {geofile}")
###############################################
# Writes output.czml used by the WebUI
###############################################
def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps):
# noinspection PyPep8Naming
def write_czml(best_point, all_the_points, ellipsedata,
plotallintersects, eps):
""" Writes output.czml used by the WebUI """
point_properties = {
"pixelSize": 5.0,
"heightReference": "CLAMP_TO_GROUND",
......@@ -594,22 +632,28 @@ def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps):
all_the_points = np.column_stack((all_the_points, scaled_time))
for x in all_the_points:
# rgb = hsvtorgb(x[-1]/3, 0.9, 0.9)
rgb = map(lambda x: int(x * 255), hsv_to_rgb(x[-1] / 3, 0.9, 0.9))
rgb = map(lambda c: int(c * 255), hsv_to_rgb(x[-1] / 3, 0.9, 0.9))
color_property = {"color": {"rgba": [*rgb, 255]}}
all_point_packets.append(Packet(id=str(x[1]) + ", " + str(x[0]),
point={**point_properties,
**color_property},
position={
"cartographicDegrees": [x[0], x[1], 0]},
))
"cartographicDegrees": [x[0],
x[1],
0]},
))
if len(best_point) > 0:
for x in best_point:
gmaps_url = f"https://www.google.com/maps/dir/?api=1&destination={x[1]},+{x[0]}&travelmode=driving"
best_point_packets.append(Packet(id=str(x[1]) + ", " + str(x[0]),
point=best_point_properties,
description=f"<a href='{gmaps_url}' target='_blank'>Google Maps Directions</a>",
position={"cartographicDegrees": [x[0], x[1], 0]}))
gmaps_url = (f"https://www.google.com/maps/dir/?api=1&"
f"destination={x[1]},+{x[0]}&travelmode=driving")
best_point_packets.append(
Packet(id=str(x[1]) + ", " + str(x[0]),
point=best_point_properties,
description=(f"<a href='{gmaps_url}' target='_blank'>"
f"Google Maps Directions</a>"),
position={"cartographicDegrees": [x[0], x[1], 0]})
)
if len(ellipsedata) > 0:
for x in ellipsedata:
......@@ -621,31 +665,39 @@ def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps):
rotation = 2 * np.pi - x[2]
rotation += np.pi / 2
# print(f"{x[2]} Inverted to: {rotation}")
# print(f"SemiMajor: {semiMajorAxis}, Semiminor: {semiMinorAxis}")
# print(f"SemiMajor: {semiMajorAxis},
# f"Semiminor: {semiMinorAxis}")
# print(f"{x[4], x[3]} is inveted")
else:
rotation = x[2]
semiMajorAxis = x[1]
semiMinorAxis = x[0]
# print(f"Not inverted: {rotation}")
# print(f"SemiMajor: {semiMajorAxis}, Semiminor: {semiMinorAxis}")
# print(f"SemiMajor: {semiMajorAxis},"
# f"Semiminor: {semiMinorAxis}")
# print(f"{x[4], x[3]} is NOT inveted")
ellipse_info = {"semiMajorAxis": semiMajorAxis,
"semiMinorAxis": semiMinorAxis, "rotation": rotation}
"semiMinorAxis": semiMinorAxis,
"rotation": rotation}
ellipse_packets.append(Packet(id=str(x[4]) + ", " + str(x[3]),
ellipse={
**ellipse_properties, **ellipse_info},
position={"cartographicDegrees": [x[3], x[4], 0]}))
**ellipse_properties,
**ellipse_info},
position={
"cartographicDegrees": [x[3],
x[4],
0]}))
document_list = [top, ]
document_list.extend(best_point_packets)
document_list.extend(all_point_packets)
document_list.extend(ellipse_packets)
return Document(document_list).dumps(separators=(',', ':'))
return Document([top] + best_point_packets + all_point_packets + ellipse_packets).dumps(separators=(',', ':'))
###############################################
# Writes receivers.czml used by the WebUI
###############################################
@get('/receivers.czml')
def write_rx_czml():
""" Writes receivers.czml used by the WebUI """
response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0',
)
......@@ -671,114 +723,140 @@ def write_rx_czml():
"height": 48,
"width": 48,
}
while not ms.rx_busy:
for index, x in enumerate(receivers):
if x.isActive and ms.receiving:
if (x.confidence > min_conf and x.power > min_power):
lob_color = green
elif (x.confidence <= min_conf and x.power > min_power):
lob_color = orange
else:
lob_color = red
lob_start_lat = x.latitude
lob_start_lon = x.longitude
lob_stop_lat, lob_stop_lon = v.direct(
lob_start_lat, lob_start_lon, x.doa, x.lob_length())
lob_packets.append(Packet(id=f"LOB-{x.station_id}-{index}",
polyline=Polyline(
material=Material(polylineOutline=PolylineOutlineMaterial(
color=Color(
rgba=lob_color),
outlineColor=Color(
rgba=[0, 0, 0, 255]),
outlineWidth=2
)),
clampToGround=True,
width=5,
positions=Position(cartographicDegrees=[
lob_start_lon, lob_start_lat, height, lob_stop_lon, lob_stop_lat, height])
)))
heading_start_lat = x.latitude
heading_start_lon = x.longitude
heading_stop_lat, heading_stop_lon = v.direct(
heading_start_lat, heading_start_lon, x.heading, heading_d)
lob_packets.append(Packet(id=f"HEADING-{x.station_id}-{index}",
polyline=Polyline(
material=PolylineMaterial(
polylineDash = PolylineDashMaterial(color=Color(
rgba=gray),
gapColor=Color(
rgba=[0, 0, 0, 0])
)),
clampToGround=True,
width=2,
positions=Position(cartographicDegrees=[
heading_start_lon, heading_start_lat, height, heading_stop_lon, heading_stop_lat, height])
)))
else:
lob_packets = []
if x.isMobile is True:
rx_icon = {"image": {"uri": "/static/flipped_car.svg"}}
# if x.heading > 0 or x.heading < 180:
# rx_icon = {"image":{"uri":"/static/flipped_car.svg"}, "rotation":math.radians(360 - x.heading + 90)}
# elif x.heading < 0 or x.heading > 180:
# rx_icon = {"image":{"uri":"/static/car.svg"}, "rotation":math.radians(360 - x.heading - 90)}
for index, x in enumerate(receivers):
if x.isActive and ms.receiving:
if x.confidence > min_conf and x.power > min_power:
lob_color = green
elif x.confidence <= min_conf and x.power > min_power:
lob_color = orange
else:
rx_icon = {"image": {"uri": "/static/tower.svg"}}
receiver_point_packets.append(Packet(id=f"{x.station_id}-{index}",
billboard={
**rx_properties, **rx_icon},
position={"cartographicDegrees": [x.longitude, x.latitude, 15]}))
response_data = Document([top] + receiver_point_packets + lob_packets).dumps(separators=(',', ':'))
# 11.01.2023::s1z: I know this is ugly and stupid,
# but we're not gonna rewrite the project anyways,
# so, whatever...
globals()["cached_receivers"] = response_data
return response_data
return globals()["cached_receivers"]
lob_color = red
lob_start_lat = x.latitude
lob_start_lon = x.longitude
lob_stop_lat, lob_stop_lon = v.direct(
lob_start_lat, lob_start_lon, x.doa, x.lob_length()
)
lob_packets.append(
Packet(
id=f"LOB-{x.station_id}-{index}",
polyline=Polyline(
material=Material(
polylineOutline=PolylineOutlineMaterial(
color=Color(rgba=lob_color),
outlineColor=Color(rgba=[0, 0, 0, 255]),
outlineWidth=2
)
),
clampToGround=True,
width=5,
positions=Position(
cartographicDegrees=[
lob_start_lon,
lob_start_lat,
height,
lob_stop_lon,
lob_stop_lat,
height
]
)
)
)
)
heading_start_lat = x.latitude
heading_start_lon = x.longitude
heading_stop_lat, heading_stop_lon = v.direct(
heading_start_lat, heading_start_lon, x.heading, heading_d)
lob_packets.append(
Packet(
id=f"HEADING-{x.station_id}-{index}",
polyline=Polyline(
material=PolylineMaterial(
polylineDash=PolylineDashMaterial(
color=Color(rgba=gray),
gapColor=Color(rgba=[0, 0, 0, 0])
)
),
clampToGround=True,
width=2,
positions=Position(
cartographicDegrees=[
heading_start_lon,
heading_start_lat,
height,
heading_stop_lon,
heading_stop_lat,
height]
)
)
)
)
else:
lob_packets = []
if x.isMobile is True:
rx_icon = {"image": {"uri": "/static/flipped_car.svg"}}
# if x.heading > 0 or x.heading < 180:
# rx_icon = {"image":{"uri":"/static/flipped_car.svg"},
# "rotation":math.radians(360 - x.heading + 90)}
# elif x.heading < 0 or x.heading > 180:
# rx_icon = {"image":{"uri":"/static/car.svg"},
# "rotation":math.radians(360 - x.heading - 90)}
else:
rx_icon = {"image": {"uri": "/static/tower.svg"}}
receiver_point_packets.append(
Packet(id=f"{x.station_id}-{index}",
billboard={**rx_properties, **rx_icon},
position={"cartographicDegrees": [x.longitude,
x.latitude, 15]})
)
document_list = [top, ]
document_list.extend(receiver_point_packets)
document_list.extend(lob_packets)
response_data = Document(document_list).dumps(separators=(',', ':'))
shared_dict["response"] = response_data
return response_data
###############################################
# Writes aoi.czml used by the WebUI
###############################################
@get("/aoi.czml")
def wr_aoi_czml():
""" Writes aoi.czml used by the WebUI """
response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
aoi_packets = []
top = Preamble(name="AOIs")
area_of_interest_properties = {
"granularity": 0.008722222,
"height": 0,
# "zIndex": 1,
"material": {
"solidColor": {
"color": {
"rgba": [0, 0, 255, 25]
}
}
},
"outline": True,
"outlineWidth": 2,
"outlineColor": {"rgba": [53, 184, 240, 255], },
},
"granularity": 0.008722222,
"height": 0,
# "zIndex": 1,
"material": {
"solidColor": {
"color": {
"rgba": [0, 0, 255, 25]
}
}
},
"outline": True,
"outlineWidth": 2,
"outlineColor": {
"rgba": [53, 184, 240, 255], },
},
exclusion_area_properties = {
"granularity": 0.008722222,
"height": 0,
# "zIndex": 0,
"material": {
"solidColor": {
"color": {
"rgba": [242, 10, 0, 25]
}
}
},
"outline": True,
"outlineWidth": 2,
"outlineColor": {"rgba": [224, 142, 0, 255], },
},
"granularity": 0.008722222,
"height": 0,
# "zIndex": 0,
"material": {
"solidColor": {
"color": {
"rgba": [242, 10, 0, 25]
}
}
},
"outline": True,
"outlineWidth": 2,
"outlineColor": {
"rgba": [224, 142, 0, 255], },
},
for x in fetch_aoi_data():
aoi = {
......@@ -796,65 +874,54 @@ def wr_aoi_czml():
"semiMinorAxis": aoi['radius'], "rotation": 0}
aoi_packets.append(Packet(id=aoi['aoi_type'] + str(aoi['uid']),
ellipse={**aoi_properties, **aoi_info},
position={"cartographicDegrees": [aoi['longitude'], aoi['latitude'], 0]}))
position={
"cartographicDegrees": [aoi['longitude'],
aoi['latitude'],
0]}))
return Document([top] + aoi_packets).dumps(separators=(',', ':'))
###############################################
# Clears the screen if debugging is off.
###############################################
def clear(debugging):
if not debugging:
# for windows
if name == 'nt':
_ = system('cls')
# for mac and linux(here, os.name is 'posix')
else:
_ = system('clear')
###############################################
# Serves static files such as CSS and JS to the
# WebUI
###############################################
@route('/static/<filepath:path>', name='static')
def server_static(filepath):
""" Serves static files such as CSS and JS to the WebUI """
response = static_file(filepath, root='./static')
response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
return response
###############################################
# Loads the main page of the WebUI
# http://[ip]:[port]/
###############################################
@get('/')
@get('/index')
@get('/cesium')
def cesium():
""" Loads the main page of the WebUI http://[ip]:[port]/ """
response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
return template('cesium.tpl',
{'access_token': access_token,
'epsilon': ms.eps,
'minpower': ms.min_power,
'minconf': ms.min_conf,
'minpoints': ms.min_samp,
'rx_state': "checked" if ms.receiving is True else "",
'intersect_state': "checked" if ms.plotintersects is True else "",
'receivers': receivers})
return template(
'cesium.tpl',
{
'access_token': access_token,
'epsilon': ms.eps,
'minpower': ms.min_power,
'minconf': ms.min_conf,
'minpoints': ms.min_samp,
'rx_state': "checked" if ms.receiving is True else "",
'intersect_state': "checked" if ms.plotintersects is True else "",
'receivers': receivers
}
)
###############################################
# GET Request to update parameters from the
# UI sliders. Not meant to be user facing.
###############################################
# noinspection SpellCheckingInspection
@get('/update')
def update_cesium():
""" GET Request to update parameters from the
UI sliders. Not meant to be user facing. """
# eps = float(request.query.eps) if request.query.eps else ms.eps
# min_samp = float(request.query.minpts) if request.query.minpts else ms.min_samp
# min_samp = (float(request.query.minpts)
# if request.query.minpts
# else ms.min_samp)
ms.min_conf = float(
request.query.minconf) if request.query.minconf else ms.min_conf
ms.min_power = float(
......@@ -873,17 +940,15 @@ def update_cesium():
return "OK"
###############################################
# Returns a JSON file to the WebUI with
# information to fill in the RX cards.
###############################################
# noinspection PyShadowingNames
@get('/rx_params')
def rx_params():
""" Returns a JSON file to the WebUI
with information to fill in the RX cards. """
all_rx = {'receivers': {}}
rx_properties = []
for index, x in enumerate(receivers):
x.update()
# x.update() # this line blocks web execution, should be async!
rx = x.receiver_dict()
rx['uid'] = index
rx_properties.append(rx)
......@@ -892,12 +957,11 @@ def rx_params():
return json.dumps(all_rx)
###############################################
# Returns a CZML file that contains intersect
# and ellipse information for Cesium.
###############################################
# noinspection SpellCheckingInspection
@get('/output.czml')
def tx_czml_out():
""" Returns a CZML file that contains intersect
and ellipse information for Cesium. """
eps = request.query.eps if request.query.eps else str(ms.eps)
min_samp = request.query.minpts if request.query.minpts else str(
ms.min_samp)
......@@ -909,20 +973,22 @@ def tx_czml_out():
plotallintersects = ms.plotintersects
response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
output = write_czml(*process_data(database_name, eps,
min_samp), plotallintersects, eps)
output = write_czml(*process_data(database_name, eps, min_samp),
plotallintersects, eps)
return str(output)
###############################################
# PUT request to update receiver variables
# from the WebUI
###############################################
# noinspection PyPep8Naming,SqlDialectInspection,PyShadowingNames
@put('/rx_params/<action>')
def update_rx(action):
""" PUT request to update receiver variables from the WebUI """
data = json.load(request.body)
if action == "new":
receiver_url = data['station_url'].replace('\n', '')
receiver_url = data['station_url'].strip()
if any(x.station_url == receiver_url for x in receivers):
logger.warning(f"The station with the url '{receiver_url}' "
"already exists!")
return redirect('/rx_params')
add_receiver(receiver_url)
elif action == "del":
index = int(data['uid'])
......@@ -946,16 +1012,15 @@ def update_rx(action):
receivers[action].update()
update_rx_table()
except IndexError:
print("I got some bad data. Doing nothing out of spite.")
logger.error("I got some bad data. Doing nothing out of spite.")
return redirect('/rx_params')
###############################################
# Returns a JSON file to the WebUI with
# information to fill in the AOI cards.
###############################################
# noinspection SpellCheckingInspection
@get('/interest_areas')
def load_interest_areas():
""" Returns a JSON file to the WebUI with
information to fill in the AOI cards. """
all_aoi = {'aois': {}}
aoi_properties = []
for x in fetch_aoi_data():
......@@ -972,13 +1037,12 @@ def load_interest_areas():
return json.dumps(all_aoi)
##########################################
# PUT request to add new AOI to DB
##########################################
# noinspection SqlNoDataSourceInspection,SqlDialectInspection
@put('/interest_areas/<action>')
def handle_interest_areas(action):
""" PUT request to add new AOI to DB """
data = json.load(request.body)
if action == "new" and not "" in data.values():
if action == "new" and "" not in data.values():
aoi_type = data['aoi_type']
lat = data['latitude']
lon = data['longitude']
......@@ -996,34 +1060,65 @@ def handle_interest_areas(action):
elif action == "purge":
conn = sqlite3.connect(database_name)
c = conn.cursor()
c.execute("SELECT aoi_type, latitude, longitude, radius FROM interest_areas WHERE uid=?", [
data['uid']])
c.execute("SELECT aoi_type, latitude, longitude, radius"
"FROM interest_areas WHERE uid=?", [data['uid']])
properties = c.fetchone()
conn.close()
purge_database(*properties)
###############################################
# Starts the Bottle webserver.
###############################################
def start_server(ipaddr="127.0.0.1", port=8080):
try:
run(host=ipaddr, port=port, quiet=True,
server=GeventWebSocketServer, debug=True)
except OSError:
print(f"Port {port} seems to be in use. Please select another port or " +
"check if another instance of DFA is already running.")
debugging = True
finish()
class BottleWebServer(threading.Thread):
""" Database Writer thread """
def __init__(self, interface="127.0.0.1", port=8080):
super().__init__(name=self.__class__.__name__, daemon=True)
self.interface = interface
self.port = port
self._is_running = False
self._is_running_lock = threading.Lock()
###############################################
# Captures DOA data and computes intersections
# if the receiver is enabled. Writes the
# intersections to the database.
###############################################
@property
def is_running(self):
""" Get _is_running safely """
with self._is_running_lock:
return self._is_running
@is_running.setter
def is_running(self, value: bool):
""" set _is_running safely """
with self._is_running_lock:
self._is_running = value
def stop(self):
""" Stop the Thread """
self.is_running = False
def start(self):
""" Start thread """
self.is_running = True
super().start()
def run(self):
""" Starts the Bottle web server. """
# noinspection PyUnresolvedReferences
from bottle.ext.websocket import GeventWebSocketServer
try:
run(host=self.interface, port=self.port, quiet=True,
server=GeventWebSocketServer, debug=True)
except OSError:
logger.error(f"Port {self.port} seems to be in use."
f"Please select another port or "
f"check if another instance of DFA is "
f"already running.")
sys.exit(1)
# noinspection PyShadowingNames,SqlDialectInspection
def run_receiver(receivers):
clear(debugging)
""" Captures DOA data and computes intersections
if the receiver is enabled.
Writes the intersections to the database."""
dots = 0
conn = sqlite3.connect(database_name)
......@@ -1031,8 +1126,7 @@ def run_receiver(receivers):
while ms.receiving:
if not debugging:
print("Receiving" + dots * '.')
print("Press Control+C to process data and exit.")
logger.info("Receiving" + dots * '.')
# Main loop to compute intersections between multiple receivers
intersect_list = np.array([]).reshape(0, 3)
......@@ -1045,32 +1139,35 @@ def run_receiver(receivers):
# We have to change this to the queue for each receiver.
rx.update()
except IOError:
print("Problem connecting to receiver.")
logger.error("Problem connecting to receiver.")
rx.d_2_last_intersection = []
time.sleep(0.1)
# for x in range(len(receivers)):
# for y in range(x):
# if x != y:
receivers_len = len(receivers)
for x in range(receivers_len):
y_receiver_id = x + 1
for y in range(y_receiver_id, receivers_len):
if (receivers[x].confidence >= ms.min_conf and
receivers[y].confidence >= ms.min_conf and
receivers[x].power >= ms.min_power and
receivers[y].power >= ms.min_power and
abs(receivers[x].doa_time - receivers[y].doa_time) <= max_age and
receivers[y].confidence >= ms.min_conf and
receivers[x].power >= ms.min_power and
receivers[y].power >= ms.min_power and
abs(receivers[x].doa_time - receivers[
y].doa_time) <= max_age and
receivers[x].frequency == receivers[y].frequency):
intersection = plot_intersects(receivers[x].latitude, receivers[x].longitude,
receivers[x].doa, receivers[y].latitude, receivers[y].longitude, receivers[y].doa)
intersection = plot_intersects(receivers[x].latitude,
receivers[x].longitude,
receivers[x].doa,
receivers[y].latitude,
receivers[y].longitude,
receivers[y].doa)
if intersection:
print(intersection)
logger.info(f"intersection: {intersection}")
receivers[x].d_2_last_intersection.append(v.haversine(
receivers[x].latitude, receivers[x].longitude, *intersection))
receivers[x].latitude, receivers[x].longitude,
*intersection))
receivers[y].d_2_last_intersection.append(v.haversine(
receivers[y].latitude, receivers[y].longitude, *intersection))
receivers[y].latitude, receivers[y].longitude,
*intersection))
intersection = list(intersection)
avg_conf = np.mean(
[receivers[x].confidence, receivers[y].confidence])
......@@ -1080,7 +1177,8 @@ def run_receiver(receivers):
intersect_list = np.concatenate(
(intersect_list, intersection), axis=0)
# 11.01.2023::s1z: I don't know if this shit-code is blocking
# or not. But if it is, you better fucking sleep
# or not.
# But if it is, you better fucking sleep
# 10ms after each cycle to prevent 100% load
# of the CPU.
time.sleep(0.001)
......@@ -1095,25 +1193,29 @@ def run_receiver(receivers):
intersect_list[:, 0:3], weights=intersect_list[:, 2], axis=0)
keep, in_aoi = check_aoi(*avg_coord[0:2])
if keep:
to_table = [receivers[x].doa_time, round(avg_coord[0], 6), round(avg_coord[1], 6),
to_table = [receivers[x].doa_time, round(avg_coord[0], 6),
round(avg_coord[1], 6),
len(intersect_list), avg_coord[2], in_aoi]
command = '''INSERT INTO intersects
command = """INSERT INTO intersects
(time, latitude, longitude, num_parents, confidence, aoi_id)
VALUES (?,?,?,?,?,?)'''
VALUES (?,?,?,?,?,?)"""
DATABASE_EDIT_Q.put((command, (to_table,), True))
DATABASE_RETURN.get(timeout=1)
# Loop to compute intersections for a single receiver and update all receivers
# Loop to compute intersections for a single receiver
# and update all receivers
for rx in receivers:
if (rx.isSingle and rx.isMobile and rx.isActive and
rx.confidence >= ms.min_conf and
rx.power >= ms.min_power and
rx.confidence >= ms.min_conf and
rx.power >= ms.min_power and
rx.doa_time >= rx.previous_doa_time + 10000):
current_doa = [rx.doa_time, rx.station_id, rx.latitude,
rx.longitude, rx.confidence, rx.doa]
min_time = rx.doa_time - 1200000 # 15 Minutes
c.execute('''SELECT latitude, longitude, confidence, lob FROM lobs
WHERE station_id = ? AND time > ?''', [rx.station_id, min_time])
c.execute("""SELECT latitude, longitude, confidence, lob
FROM lobs
WHERE station_id = ? AND time > ?""",
[rx.station_id, min_time])
lob_array = c.fetchall()
current_time = current_doa[0]
lat_rxa = current_doa[2]
......@@ -1133,7 +1235,8 @@ def run_receiver(receivers):
if (spacial_diversity > min_diversity and
abs(doa_rxa - doa_rxb) > 5):
intersection = plot_intersects(lat_rxa, lon_rxa,
doa_rxa, lat_rxb, lon_rxb, doa_rxb)
doa_rxa, lat_rxb,
lon_rxb, doa_rxb)
if intersection:
intersection = list(intersection)
avg_conf = np.mean([conf_rxa, conf_rxb])
......@@ -1141,15 +1244,19 @@ def run_receiver(receivers):
keep, in_aoi = check_aoi(*intersection[0:2])
if keep:
keep_count += 1
to_table = [current_time, round(intersection[0], 5), round(intersection[1], 5),
to_table = [current_time,
round(intersection[0], 5),
round(intersection[1], 5),
1, intersection[2], in_aoi]
command = '''INSERT INTO intersects
(time, latitude, longitude, num_parents, confidence, aoi_id)
VALUES (?,?,?,?,?,?)'''
DATABASE_EDIT_Q.put(
(command, (to_table,), True))
command = """INSERT INTO intersects
(time, latitude, longitude,
num_parents, confidence,
aoi_id)
VALUES (?,?,?,?,?,?)"""
DATABASE_EDIT_Q.put((command, (to_table,),
True))
DATABASE_RETURN.get(timeout=1)
print(f"Computed and kept {keep_count} intersections.")
logger.info(f"Computed and kept {keep_count} intersections.")
command = "INSERT INTO lobs VALUES (?,?,?,?,?,?)"
DATABASE_EDIT_Q.put((command, [current_doa, ], True))
......@@ -1172,15 +1279,13 @@ def run_receiver(receivers):
dots = 1
else:
dots += 1
clear(debugging)
conn.close()
###############################################
# Checks if intersection should be kept or not
###############################################
# noinspection SqlDialectInspection
def check_aoi(lat, lon):
""" Checks if intersection should be kept or not """
keep_list = []
in_aoi = None
conn = sqlite3.connect(database_name)
......@@ -1215,73 +1320,83 @@ def check_aoi(lat, lon):
return keep, in_aoi
###############################################
# Adds a new receiver to the program, saves it
# in the database.
###############################################
def add_receiver(receiver_url):
def fetch_first_or_none(query):
""" Safe fetch one """
result = query.fetchone()
if result is not None and len(result) > 0:
return result[0]
return None
# noinspection SqlDialectInspection
def add_receiver(receiver_url, station_name=None):
""" Adds a new receiver to the program, saves it in the database. """
conn = sqlite3.connect(database_name)
c = conn.cursor()
try:
if any(x.station_url == receiver_url for x in receivers):
print("Duplicate receiver, ignoring.")
logger.warning("Duplicate receiver, ignoring.")
else:
receivers.append(receiver(receiver_url))
new_rx = receivers[-1].receiver_dict()
to_table = [new_rx['station_id'], new_rx['station_url'], new_rx['auto'],
new_rx['mobile'], new_rx['single'], new_rx['latitude'], new_rx['longitude']]
new_receiver = receiver(receiver_url, station_name)
receivers.append(new_receiver)
new_rx = new_receiver.receiver_dict()
to_table = [new_rx['station_id'], new_rx['station_url'],
new_rx['auto'],
new_rx['mobile'], new_rx['single'], new_rx['latitude'],
new_rx['longitude']]
command = "INSERT OR IGNORE INTO receivers VALUES (?,?,?,?,?,?,?)"
DATABASE_EDIT_Q.put((command, [to_table, ], True))
DATABASE_RETURN.get(timeout=1)
DATABASE_EDIT_Q.put(("done", None, False))
mobile = c.execute("SELECT isMobile FROM receivers WHERE station_id = ?",
[new_rx['station_id']]).fetchone()[0]
single = c.execute("SELECT isSingle FROM receivers WHERE station_id = ?",
[new_rx['station_id']]).fetchone()[0]
receivers[-1].isMobile = bool(mobile)
receivers[-1].isSingle = bool(single)
print("Created new DF Station at " + receiver_url)
mobile = fetch_first_or_none(
c.execute("SELECT isMobile FROM receivers "
"WHERE station_id = ?", [new_rx['station_id']])
)
single = fetch_first_or_none(
c.execute("SELECT isSingle FROM receivers "
"WHERE station_id = ?", [new_rx['station_id']])
)
if mobile is not None:
new_receiver.isMobile = bool(mobile)
if single is not None:
new_receiver.isSingle = bool(single)
logger.info("Created new DF Station at " + receiver_url)
except AttributeError:
pass
conn.close()
###############################################
# Reads receivers from the database into the
# program.
###############################################
# noinspection PyShadowingNames
def read_rx_table():
""" Reads receivers from the database into the program. """
conn = sqlite3.connect(database_name)
c = conn.cursor()
try:
c.execute("SELECT station_url FROM receivers")
c.execute("SELECT station_url, station_id FROM receivers")
rx_list = c.fetchall()
for x in rx_list:
receiver_url = x[0].replace('\n', '')
add_receiver(receiver_url)
for data in rx_list:
station_url, station_id = [param.strip() for param in data]
add_receiver(station_url, station_id)
except sqlite3.OperationalError:
rx_list = []
conn.close()
###############################################
# Updates the database with any changes made to
# the receivers.
###############################################
# noinspection SqlDialectInspection
def update_rx_table():
""" Updates the database with any changes made to the receivers. """
for item in receivers:
rx = item.receiver_dict()
to_table = [rx['auto'], rx['mobile'], rx['single'],
rx['latitude'], rx['longitude'], rx['station_id']]
command = '''UPDATE receivers SET
isAuto=?,
isMobile=?,
isSingle=?,
latitude=?,
longitude=?
WHERE station_id = ?'''
command = """UPDATE receivers SET isAuto=?,
isMobile=?,
isSingle=?,
latitude=?,
longitude=?
WHERE station_id = ?"""
DATABASE_EDIT_Q.put((command, [to_table, ], True))
# try:
DATABASE_RETURN.get(timeout=1)
......@@ -1290,10 +1405,9 @@ def update_rx_table():
DATABASE_EDIT_Q.put(("done", None, False))
###############################################
# Updates the database with new interest areas.
###############################################
# noinspection SqlDialectInspection
def add_aoi(aoi_type, lat, lon, radius):
""" Updates the database with new interest areas. """
conn = sqlite3.connect(database_name)
c = conn.cursor()
......@@ -1307,10 +1421,8 @@ def add_aoi(aoi_type, lat, lon, radius):
DATABASE_EDIT_Q.put(("done", None, False))
#########################################
# Read all the AOIs from the DB
#########################################
def fetch_aoi_data():
""" Read all the AOIs from the DB """
conn = sqlite3.connect(database_name)
c = conn.cursor()
c.execute('SELECT * FROM interest_areas')
......@@ -1319,74 +1431,114 @@ def fetch_aoi_data():
return aoi_list
###############################################
# One thread responsible for all database write
# operations.
###############################################
def database_writer():
conn = sqlite3.connect(database_name)
c = conn.cursor()
c.execute('''CREATE TABLE IF NOT EXISTS receivers (
station_id TEXT UNIQUE,
station_url TEXT,
isAuto INTEGER,
isMobile INTEGER,
isSingle INTEGER,
latitude REAL,
longitude REAL)
''')
c.execute('''CREATE TABLE IF NOT EXISTS interest_areas (
uid INTEGER,
aoi_type TEXT,
latitude REAL,
longitude REAL,
radius INTEGER)
''')
c.execute('''CREATE TABLE IF NOT EXISTS intersects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
time INTEGER,
latitude REAL,
longitude REAL,
num_parents INTEGER,
confidence INTEGER,
aoi_id INTEGER)''')
c.execute('''CREATE TABLE IF NOT EXISTS lobs (time INTEGER,
station_id TEXT,
latitude REAL,
longitude REAL,
confidence INTEGER,
lob REAL)''')
conn.commit()
while True:
# items should be list of lists
command, items, reply = DATABASE_EDIT_Q.get()
if command == "done":
conn.commit()
elif command == "close":
conn.commit()
conn.close()
if reply:
DATABASE_RETURN.put(True)
break
else:
c.executemany(command, items)
if reply:
DATABASE_RETURN.put(True)
class DatabaseWriter(threading.Thread):
""" Database Writer thread """
def __init__(self, db_name):
super().__init__(name=self.__class__.__name__, daemon=True)
self.database_name = db_name
self._is_running = False
self._is_running_lock = threading.Lock()
@property
def is_running(self):
""" Get _is_running safely """
with self._is_running_lock:
return self._is_running
@is_running.setter
def is_running(self, value: bool):
""" set _is_running safely """
with self._is_running_lock:
self._is_running = value
def stop(self):
""" Stop the Thread """
self.is_running = False
def start(self):
""" Start thread """
self.is_running = True
super().start()
# noinspection SqlDialectInspection
def run(self):
""" One thread responsible for all database write operations. """
conn = sqlite3.connect(self.database_name)
c = conn.cursor()
c.execute("""CREATE TABLE IF NOT EXISTS receivers (
station_id TEXT UNIQUE,
station_url TEXT,
isAuto INTEGER,
isMobile INTEGER,
isSingle INTEGER,
latitude REAL,
longitude REAL)
""")
c.execute("""CREATE TABLE IF NOT EXISTS interest_areas (
uid INTEGER,
aoi_type TEXT,
latitude REAL,
longitude REAL,
radius INTEGER)
""")
c.execute("""CREATE TABLE IF NOT EXISTS intersects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
time INTEGER,
latitude REAL,
longitude REAL,
num_parents INTEGER,
confidence INTEGER,
aoi_id INTEGER)""")
c.execute("""CREATE TABLE IF NOT EXISTS lobs (time INTEGER,
station_id TEXT,
latitude REAL,
longitude REAL,
confidence INTEGER,
lob REAL)""")
conn.commit()
while self.is_running:
# items should be list of lists
command, items, reply = DATABASE_EDIT_Q.get()
if command == "done":
conn.commit()
elif command == "close":
conn.commit()
conn.close()
if reply:
DATABASE_RETURN.put(True)
break
else:
c.executemany(command, items)
if reply:
DATABASE_RETURN.put(True)
time.sleep(0.001)
###############################################
# Thangs to do before closing the program.
###############################################
def finish():
clear(debugging)
print("Processing, please wait.")
""" Thangs to do before closing the program. """
logger.info("Processing df-aggregator clean-up tasks")
ms.receiving = False
update_rx_table()
DATABASE_EDIT_Q.put(("close", None, True))
DATABASE_RETURN.get(timeout=1)
if geofile is not None:
write_geojson(*process_data(database_name)[:2])
kill(getpid(), signal.SIGTERM)
# kill(getpid(), signal.SIGTERM)
logger.info("clean-up has done")
def stop_server(server):
""" Stop server and execute finish gracefuly """
def wr(_signum, _frame):
""" We need a wraper to dynamicaly set server """
server.stop()
server.join()
finish()
return wr
if __name__ == '__main__':
......@@ -1401,32 +1553,54 @@ if __name__ == '__main__':
help="List of receiver URLs", metavar="FILE")
parser.add_option("-g", "--geofile", dest="geofile",
help="GeoJSON Output File", metavar="FILE")
parser.add_option("-e", "--epsilon", dest="eps", help="Max Clustering Distance, Default \"auto\".",
parser.add_option("-e", "--epsilon", dest="eps",
help="Max Clustering Distance, Default \"auto\".",
metavar="NUMBER or \"auto\"", default="auto")
parser.add_option("-c", "--confidence", dest="conf", help="Minimum confidence value, default 10",
parser.add_option("-c", "--confidence", dest="conf",
help="Minimum confidence value, default 10",
metavar="NUMBER", type="int", default=10)
parser.add_option("-p", "--power", dest="pwr", help="Minimum power value, default 10",
parser.add_option("-p", "--power", dest="pwr",
help="Minimum power value, default 10",
metavar="NUMBER", type="int", default=10)
parser.add_option("-m", "--min-samples", dest="minsamp", help="Minimum samples per cluster. Default: \"auto\"",
parser.add_option("-m", "--min-samples", dest="minsamp",
help="Minimum samples per cluster. Default: \"auto\"",
metavar="NUMBER or \"auto\"", default="auto")
parser.add_option("--plot_intersects", dest="plotintersects", help="""Plots all the intersect points in a cluster.
Only applies when clustering is turned on. This creates larger CZML files.""", action="store_true")
parser.add_option("-o", "--offline", dest="disable", help="Starts program with receiver turned off.",
parser.add_option("--plot_intersects", dest="plotintersects",
help=("Plots all the intersect points in a cluster. "
"Only applies when clustering is turned on. "
"This creates larger CZML files."),
action="store_true")
parser.add_option("-o", "--offline", dest="disable",
help="Starts program with receiver turned off.",
action="store_false", default=True)
parser.add_option("--access_token", dest="token_file",
help="Cesium Access Token File", metavar="FILE")
parser.add_option("--ip", dest="ipaddr", help="IP Address to serve from. Default 127.0.0.1",
parser.add_option("--ip", dest="ipaddr",
help="IP Address to serve from. Default 127.0.0.1",
metavar="IP ADDRESS", type="str", default="127.0.0.1")
parser.add_option("--port", dest="port", help="Port number to serve from. Default 8080",
parser.add_option("--port", dest="port",
help="Port number to serve from. Default 8080",
metavar="NUMBER", type="int", default=8080)
parser.add_option("--debug", dest="debugging", help="Does not clear the screen. Useful for seeing errors and warnings.",
parser.add_option("--debug", dest="debugging",
help="Does not clear the screen. "
"Useful for seeing errors and warnings.",
action="store_true")
(options, args) = parser.parse_args()
# INIT LOGGING
logging_level = logging.DEBUG
if not options.debugging: # Limit some spam
# noinspection SpellCheckingInspection
logging.getLogger("geventwebsocket.handler").level = logging.WARNING
logging.getLogger("asyncio").level = logging.INFO
logging_level = logging.INFO
init_logging(level=logging_level)
#
mandatories = ['database_name']
for m in mandatories:
if options.__dict__[m] is None:
print("You forgot an arguement")
logger.error("You forgot an arguement")
parser.print_help()
exit(-1)
......@@ -1447,41 +1621,38 @@ if __name__ == '__main__':
else:
access_token = None
web = threading.Thread(target=start_server,
args=(options.ipaddr, options.port))
web.daemon = True
web = BottleWebServer(options.ipaddr, options.port)
web.start()
dbwriter = threading.Thread(target=database_writer)
dbwriter.daemon = True
dbwriter.start()
db_writer = DatabaseWriter(database_name)
db_writer.start()
try:
###############################################
# Reds receivers from the database first, then
# loads receivers in from a user provided list.
###############################################
read_rx_table()
if rx_file:
with open(rx_file, "r") as file2:
receiver_list = file2.readlines()
for x in receiver_list:
receiver_url = x.replace('\n', '')
add_receiver(receiver_url)
###############################################
# Run the main loop!
###############################################
while True:
if ms.receiving:
run_receiver(receivers)
clear(debugging)
if not debugging:
print("Receiver Paused")
# 11.01.2023::s1z: I don't know wtf is this crap, but sleep 1 sec,
# You gotta be kidding me.
# Changed fro 1 sec to 10 ms.
time.sleep(0.01)
except KeyboardInterrupt:
finish()
###############################################
# Reds receivers from the database first, then
# loads receivers in from a user provided list.
###############################################
read_rx_table()
if rx_file:
with open(rx_file, "r") as file2:
for receiver_url in file2.readlines():
add_receiver(receiver_url.strip())
###############################################
# Run the main loop!
###############################################
# while True:
# if ms.receiving:
# run_receiver(receivers)
# if not debugging:
# print("Receiver Paused")
# # 11.01.2023::s1z: I don't know wtf is this crap,
# # but sleep 1 sec,
# # You gotta be kidding me.
# # Changed from 1 sec to 10 ms.
# time.sleep(0.001)
ether_service = EtherService(web, db_writer, receivers, shared_dict)
signal.signal(signal.SIGINT, stop_server(ether_service))
signal.signal(signal.SIGTERM, stop_server(ether_service))
ether_service.start()
ether_service.join()
requirements.txt
......@@ -5,3 +5,8 @@ numpy>=1.13.3
lxml>=4.2.1
czml3>=0.5.4
scikit_learn>=0.23.2
# ether requirements
absl-py==1.4.0
aiohttp==3.8.4
simplejson==3.18.4
utils/ether_service/entities/receiver.py 0 → 100644
""" This is a Protocol to wirk with obsolete DF-Aggregator objects """
from typing import Protocol
class Receiver(Protocol):
""" Receiver object """
...
def update(self, first_run=False):
""" Some update logic """
...
def receiver_dict(self):
""" Receiver representation in json format """
...
def lob_length(self):
""" Some magic here, i suppose. We skip it for now.
There's a bit of hardcode with 200... """
...
utils/ether_service/ether_service.py 0 → 100644
""" Ether web service """
import asyncio
import logging
from threading import Thread, Lock
from typing import List, Dict
from aiohttp import web
from utils.ether_service.entities.receiver import Receiver
from utils.ether_service.handlers.api.v1.ws import WSHandler
from utils.run_app_fixed import create_task
logger = logging.getLogger(__name__)
class EtherService(Thread):
""" Ether Service. This is a thread that runs aiohttp.
Main purpose is:
1. Receive data from KrSPIs.
2. Do math on the background.
3. Update global variables for the obsolete web service."""
def __init__(self, web_service: Thread, db_writer: Thread,
receivers: List[Receiver], shared_dict: Dict,
*,
interface="0.0.0.0", port=8090):
super().__init__(name=self.__class__.__name__, daemon=True)
self.web_service = web_service
self.db_writer = db_writer
self.receivers = receivers
self.shared_dict = shared_dict
self._is_running = False
self._is_running_lock = Lock()
self.interface = interface
self.port = port
self.io_loop = asyncio.get_event_loop()
self.web_app = web.Application()
self.web_app.add_routes([
WSHandler().compile(),
])
self.main_task = None
@property
def is_running(self):
""" Get _is_running safely """
with self._is_running_lock:
return self._is_running
@is_running.setter
def is_running(self, value: bool):
""" set _is_running safely """
with self._is_running_lock:
self._is_running = value
def run(self):
""" Thread routine """
self.is_running = True
self.main_task = create_task(self.web_app, host=self.interface,
port=self.port, loop=self.io_loop,
handle_signals=False,
print=lambda x: x)
logger.info("Server has started")
self.io_loop.run_forever()
logger.info("Server has stopped")
def start(self):
""" Start the app """
logger.info("Server is starting")
super().start()
def stop(self):
""" Stop the app """
logger.info("Server is stopping")
self.is_running = False
# add web_server.stop()
# add db_writer.stop()
self.io_loop.call_soon_threadsafe(self.main_task.cancel)
self.io_loop.call_soon_threadsafe(self.io_loop.stop)
utils/ether_service/handlers/api/v1/ws.py 0 → 100644
""" Websocket handler """
import base64
import sys
from typing import Any, Awaitable, Union, Tuple, Optional
import aiohttp
import uuid
from absl import logging
from aiohttp import WSMessage
from aiohttp.web_exceptions import HTTPForbidden
from aiohttp.web_request import Request
from aiohttp.web_response import Response
from aiohttp.web_ws import WebSocketResponse
from multidict import CIMultiDictProxy
from ...base_handler import BaseHandler
class AuthType:
""" Supported auth types """
BASIC = "basic"
BEARER = "bearer"
async def auth_basic(credentials: str) -> bool:
""" Auth using basic credentials """
# noinspection PyBroadException
try:
data = base64.b64decode(credentials.encode("utf-8")).decode("utf-8")
login, password = data.split(":")
if login == "kraken" and password == "kraken":
return True
except Exception:
logging.error("auth_basic error:", exc_info=sys.exc_info())
return False
def parse_auth_header(headers: "CIMultiDictProxy[str]")\
-> Tuple[Optional[str], Optional[str]]:
""" Parse auth header, return AuthType and AuthData """
auth_header = headers.get("Authorization") or None
if auth_header is not None:
try:
auth_type, auth_data = auth_header.split(" ")
return auth_type, auth_data
except ValueError:
logging.error(f"Could not parse Auth Header: '{auth_header}'")
return None, None
async def authenticate(request: Request) -> bool:
""" Authenticate request """
auth_type, auth_data = parse_auth_header(request.headers)
if auth_type is not None and auth_data is not None:
logging.info(f"auth_type and auth_data are fine, handling")
if auth_type.lower() == AuthType.BASIC:
logging.info(f"Handling Basic auth")
return await auth_basic(auth_data)
elif auth_data.lower() == AuthType.BEARER:
logging.warning(f"{AuthType.BEARER} is not implemented!")
return False
else:
logging.warning(f"{auth_type} is not supported!")
logging.warning(f"Either, auth_type or auth_data is null")
return False
def auth_required(f) -> Any:
""" Check if user is auth """
async def wr(self, request: Request, *args, **kwargs)\
-> Awaitable[Union[WebSocketResponse, Response]]:
""" wrapper """
is_auth = await authenticate(request) or self.PATH.find("?") >= 0
if is_auth:
return await f(self, request, *args, **kwargs)
raise HTTPForbidden()
return wr
class WSHandler(BaseHandler):
""" DoA Handler """
METHOD = "GET"
PATH = "/api/v1/ws"
ws = None
connection_id = None
@staticmethod
async def handle_binary(data: bytes) -> None:
""" handle web socket data """
logging.info(f"Binary data received: {data}")
async def handle_message(self, msg: WSMessage) -> None:
""" Handle incoming websocket packet """
if msg.type == aiohttp.WSMsgType.ERROR:
ex = self.ws.exception()
exc_info = (ex.__class__, ex, ex.__traceback__)
logging.error("Ws connection closed", exc_info=exc_info)
elif msg.type == aiohttp.WSMsgType.BINARY:
await self.handle_binary(msg.data)
elif msg.type == aiohttp.WSMsgType.TEXT:
await self.ws.send_str(msg.data)
@auth_required
async def handler(self, request: Request) -> WebSocketResponse:
""" The handler """
connection_id = uuid.uuid4()
logging.info(f"WS opened. connection_id: '{connection_id}'")
self.ws = ws = WebSocketResponse()
await ws.prepare(request)
logging.info(f"WS prepared. connection_id: '{connection_id}'")
async for msg in ws:
# noinspection PyTypeChecker
await self.handle_message(msg)
logging.info(f"WS closed. connection_id: '{connection_id}'")
return ws
utils/ether_service/handlers/base_handler.py 0 → 100644
""" This is a base handler """
import logging
from aiohttp import web
from aiohttp.web_request import Request
from aiohttp.web_response import Response
class BaseHandler:
""" API V1 Health Check """
METHOD = None
PATH = None
async def handler(self, _: Request) -> Response:
""" The handler """
raise NotImplementedError()
def compile(self):
""" compile into the route """
logger = logging.getLogger(self.__module__)
logger.info(f"{self.METHOD}:{self.PATH}")
return web.route(self.METHOD, self.PATH, self.handler)
def _get_name(self):
""" Get handler name """
return f"{self.__module__}.{self.__class__.__name__}"
utils/log.py 0 → 100644
from __future__ import absolute_import
from __future__ import unicode_literals
import logging
def init_logging(filename=None, level=None):
""" init logging on the app level """
logging_config = {"format": "%(asctime)-23s %(levelname)8s::"
"%(name)s::%(funcName)s:"
"%(message)s",
"level": level or logging.DEBUG}
if filename is not None:
logging_config["filename"] = filename
logging.getLogger().handlers = []
logging.basicConfig(**logging_config)
utils/run_app_fixed.py 0 → 100644
""" Default run_app uses method run_until_complete,
which is awful and wrongn """
import asyncio
import logging
import socket
from asyncio import all_tasks, Task
from ssl import SSLContext
from typing import Union, Awaitable, Optional
from aiohttp.abc import Application, AbstractAccessLogger
from aiohttp.log import access_logger
from aiohttp.web import HostSequence, _run_app, _cancel_tasks
from typing import (
Any,
Awaitable,
Callable,
Iterable as TypingIterable,
List,
Optional,
Set,
Type,
Union,
cast,
)
from aiohttp.web_log import AccessLogger
from aiohttp.web_runner import GracefulExit
def create_task(
app: Union[Application, Awaitable[Application]],
*,
host: Optional[Union[str, HostSequence]] = None,
port: Optional[int] = None,
path: Optional[str] = None,
sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None,
shutdown_timeout: float = 60.0,
keepalive_timeout: float = 75.0,
ssl_context: Optional[SSLContext] = None,
print: Callable[..., None] = print,
backlog: int = 128,
access_log_class: Type[AbstractAccessLogger] = AccessLogger,
access_log_format: str = AccessLogger.LOG_FORMAT,
access_log: Optional[logging.Logger] = access_logger,
handle_signals: bool = True,
reuse_address: Optional[bool] = None,
reuse_port: Optional[bool] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
) -> Task:
"""Run an app locally"""
if loop is None:
loop = asyncio.new_event_loop()
# Configure if and only if in debugging mode and using the default logger
if loop.get_debug() and access_log and access_log.name == "aiohttp.access":
if access_log.level == logging.NOTSET:
access_log.setLevel(logging.DEBUG)
if not access_log.hasHandlers():
access_log.addHandler(logging.StreamHandler())
main_task = loop.create_task(
_run_app(
app,
host=host,
port=port,
path=path,
sock=sock,
shutdown_timeout=shutdown_timeout,
keepalive_timeout=keepalive_timeout,
ssl_context=ssl_context,
print=print,
backlog=backlog,
access_log_class=access_log_class,
access_log_format=access_log_format,
access_log=access_log,
handle_signals=handle_signals,
reuse_address=reuse_address,
reuse_port=reuse_port,
)
)
return main_task
def run_app(
app: Union[Application, Awaitable[Application]],
*,
host: Optional[Union[str, HostSequence]] = None,
port: Optional[int] = None,
path: Optional[str] = None,
sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None,
shutdown_timeout: float = 60.0,
keepalive_timeout: float = 75.0,
ssl_context: Optional[SSLContext] = None,
print: Callable[..., None] = print,
backlog: int = 128,
access_log_class: Type[AbstractAccessLogger] = AccessLogger,
access_log_format: str = AccessLogger.LOG_FORMAT,
access_log: Optional[logging.Logger] = access_logger,
handle_signals: bool = True,
reuse_address: Optional[bool] = None,
reuse_port: Optional[bool] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
) -> None:
"""Run an app locally"""
main_task = create_task(app, host=host, port=port, path=path, sock=sock,
shutdown_timeout=shutdown_timeout,
keepalive_timeout=keepalive_timeout,
ssl_context=ssl_context, print=print,
backlog=backlog,
access_log_class=access_log_class,
access_log_format=access_log_format,
access_log=access_log,
handle_signals=handle_signals,
reuse_address=reuse_address,
reuse_port=reuse_port, loop=loop)
try:
asyncio.set_event_loop(loop)
loop.run_forever()
except (GracefulExit, KeyboardInterrupt): # pragma: no cover
pass
finally:
_cancel_tasks({main_task}, loop)
_cancel_tasks(all_tasks(loop), loop)
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment