Skip to content

D
df-aggregator
Commit 5b9c96dc by Oleksandr Barabash
Options

init commit. Removed update on receivers as it's going to be obsolete

parent 5eccee51
Showing with 1064 additions and 494 deletions
.gitignore
...@@ -14,3 +14,7 @@ static/node_modules ...@@ -14,3 +14,7 @@ static/node_modules
static/cesium-kml-czml-editor static/cesium-kml-czml-editor
package-lock.json package-lock.json
package.json package.json
*.pyc
**.pyc
.venv
.idea
df-aggregator.py
#!/usr/bin/env python3 """ DF-Agg """
# df-aggregator, networked radio direction finding software. # df-aggregator, networked radio direction finding software.
# Copyright (C) 2020 Corey Koval # Copyright (C) 2020 Corey Koval
# #
...@@ -15,6 +14,9 @@ ...@@ -15,6 +14,9 @@
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import sys
import uuid
import vincenty as v import vincenty as v
import numpy as np import numpy as np
...@@ -26,24 +28,35 @@ import signal ...@@ -26,24 +28,35 @@ import signal
import json import json
from colorsys import hsv_to_rgb from colorsys import hsv_to_rgb
from optparse import OptionParser from optparse import OptionParser
from os import system, name, kill, getpid
from lxml import etree from lxml import etree
from sklearn.cluster import DBSCAN from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler, minmax_scale from sklearn.preprocessing import StandardScaler, minmax_scale
from geojson import MultiPoint, Feature, FeatureCollection from geojson import MultiPoint, Feature, FeatureCollection
from czml3 import Packet, Document, Preamble from czml3 import Packet, Document, Preamble
from czml3.properties import Position, Polyline, PolylineMaterial, PolylineOutlineMaterial, PolylineDashMaterial, Color, Material from czml3.properties import (
Position, Polyline, PolylineMaterial, PolylineOutlineMaterial,
PolylineDashMaterial, Color, Material
)
from multiprocessing import Process, Queue from multiprocessing import Process, Queue
from bottle import route, run, request, get, put, response, redirect, template, static_file from bottle import (route, run, request, get, put, response,
from bottle.ext.websocket import GeventWebSocketServer, websocket redirect, template, static_file)
from sys import version_info from sys import version_info
if (version_info.major != 3 or version_info.minor < 6):
print("Looks like you're running python version " + from utils.ether_service.ether_service import EtherService
str(version_info.major) + "." + from utils.log import init_logging
str(version_info.minor) + ", which is no longer supported.")
print("Your python version is out of date, please update to 3.6 or newer.")
quit() logger = logging.getLogger(__name__)
if version_info.major != 3 or version_info.minor < 6:
logger.error("Looks like you're running python version "
f"{version_info.major}.{version_info.minor} "
"which is no longer supported.")
logger.error("Your python version is out of date, "
"please update to 3.6 or newer.")
sys.exit(1)
DBSCAN_Q = Queue() DBSCAN_Q = Queue()
...@@ -55,38 +68,66 @@ d = 40000 # draw distance of LOBs in meters ...@@ -55,38 +68,66 @@ d = 40000 # draw distance of LOBs in meters
heading_d = 20000 heading_d = 20000
max_age = 5000 max_age = 5000
receivers = [] receivers = []
cached_receivers = [] shared_dict = dict(response=None)
###############################################
# Stores settings realted to intersect capture # noinspection PyPep8Naming,SpellCheckingInspection
# and post-processing.
###############################################
class math_settings: class math_settings:
""" Stores settings related to intersect capture and post-processing. """
def __init__(self, eps, min_samp, conf, power): def __init__(self, eps, min_samp, conf, power):
self.eps = eps self.eps = eps
self.min_samp = min_samp self.min_samp = min_samp
self.min_conf = conf self.min_conf = conf
self.min_power = power self.min_power = power
rx_busy = False
rx_busy = True
receiving = True receiving = True
plotintersects = False plotintersects = False
################################################ # noinspection PyPep8Naming
# Stores all variables pertaining to a reveiver.
# Also updates receiver variable upon request.
################################################
class receiver: class receiver:
def __init__(self, station_url): """ Stores all variables pertaining to a reveiver.
Also updates receiver variable upon request."""
latitude = 0.0
longitude = 0.0
heading = 0.0
raw_doa = 0.0
doa = 0.0
frequency = 0.0
power = 0.0
confidence = 0
doa_time = 0
isMobile = False
isSingle = False
previous_doa_time = 0
last_processed_at = 0
d_2_last_intersection = [d]
def __init__(self, station_url, station_name=None):
self.station_url = station_url self.station_url = station_url
self.station_id = station_name or f"Unknown-{uuid.uuid4()}"
self.isAuto = True self.isAuto = True
self.isActive = True self.isActive = True
self.flipped = False self.flipped = False
self.inverted = True self.inverted = True
self.update(first_run=True)
# self.update(first_run=True) # Don't do this! Never, ever!
# init default values
self.latitude = 0.0
self.longitude = 0.0
self.heading = 0.0
self.raw_doa = 0.0
self.doa = 0.0
self.frequency = 0.0
self.power = 0.0
self.confidence = 0
self.doa_time = 0
# Updates receiver from the remote URL # Updates receiver from the remote URL
def update(self, first_run=False): def update(self, first_run=False):
""" update """
try: try:
xml_contents = etree.parse(self.station_url) xml_contents = etree.parse(self.station_url)
xml_station_id = xml_contents.find('STATION_ID') xml_station_id = xml_contents.find('STATION_ID')
...@@ -117,8 +158,6 @@ class receiver: ...@@ -117,8 +158,6 @@ class receiver:
self.power = float(xml_power.text) self.power = float(xml_power.text)
xml_conf = xml_contents.find('CONF') xml_conf = xml_contents.find('CONF')
self.confidence = int(xml_conf.text) self.confidence = int(xml_conf.text)
except KeyboardInterrupt:
finish()
except Exception as ex: except Exception as ex:
# TODO(s1z): This is a bullshit handler! # TODO(s1z): This is a bullshit handler!
# Has to be changed to appropriate actions depend on # Has to be changed to appropriate actions depend on
...@@ -146,46 +185,40 @@ class receiver: ...@@ -146,46 +185,40 @@ class receiver:
self.confidence = 0 self.confidence = 0
self.doa_time = 0 self.doa_time = 0
self.isActive = False self.isActive = False
print(ex) exc_info = (type(ex), ex, ex.__traceback__)
print( logger.error("Error occurred", exc_info=exc_info)
f"Problem connecting to {self.station_url}, receiver deactivated. Reactivate in WebUI.") logger.error(f"Problem connecting to {self.station_url}, "
# raise IOError f"receiver deactivated. Reactivate in WebUI.")
# Returns receivers properties as a dict, useful for passing data to the WebUI
def receiver_dict(self): def receiver_dict(self):
return ({'station_id': self.station_id, 'station_url': self.station_url, """ Returns receivers properties as a dict,
'latitude': self.latitude, 'longitude': self.longitude, 'heading': self.heading, useful for passing data to the WebUI"""
'doa': self.doa, 'frequency': self.frequency, 'power': self.power, return {'station_id': self.station_id,
'confidence': self.confidence, 'doa_time': self.doa_time, 'mobile': self.isMobile, 'station_url': self.station_url,
'active': self.isActive, 'auto': self.isAuto, 'inverted': self.inverted, 'latitude': self.latitude,
'single': self.isSingle}) 'longitude': self.longitude,
'heading': self.heading,
'doa': self.doa,
'frequency': self.frequency,
'power': self.power,
'confidence': self.confidence,
'doa_time': self.doa_time,
'mobile': self.isMobile,
'active': self.isActive,
'auto': self.isAuto,
'inverted': self.inverted,
'single': self.isSingle}
def lob_length(self): def lob_length(self):
""" lob_length """
if self.d_2_last_intersection: if self.d_2_last_intersection:
return round(max(self.d_2_last_intersection)) + 200 return round(max(self.d_2_last_intersection)) + 200
else: else:
return d return d
latitude = 0.0
longitude = 0.0
heading = 0.0
raw_doa = 0.0
doa = 0.0
frequency = 0.0
power = 0.0
confidence = 0
doa_time = 0
isMobile = False
isSingle = False
previous_doa_time = 0
last_processed_at = 0
d_2_last_intersection = [d]
###############################################
# Converts Lat/Lon to polar coordinates
###############################################
def plot_polar(lat_a, lon_a, lat_a2, lon_a2): def plot_polar(lat_a, lon_a, lat_a2, lon_a2):
""" Converts Lat/Lon to polar coordinates """
# Convert points in great circle 1, degrees to radians # Convert points in great circle 1, degrees to radians
p1_lat1_rad = math.radians(lat_a) p1_lat1_rad = math.radians(lat_a)
p1_long1_rad = math.radians(lon_a) p1_long1_rad = math.radians(lon_a)
...@@ -206,7 +239,8 @@ def plot_polar(lat_a, lon_a, lat_a2, lon_a2): ...@@ -206,7 +239,8 @@ def plot_polar(lat_a, lon_a, lat_a2, lon_a2):
##################################################### #####################################################
# Find line of intersection between two great circles # Find line of intersection between two great circles
##################################################### #####################################################
def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=100000): def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b,
max_distance=100000):
# plot another point on the lob # plot another point on the lob
# v.direct(lat_a, lon_a, doa_a, d) # v.direct(lat_a, lon_a, doa_a, d)
# returns (lat_a2, lon_a2) # returns (lat_a2, lon_a2)
...@@ -223,11 +257,12 @@ def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=10000 ...@@ -223,11 +257,12 @@ def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=10000
# Find line of intersection between two planes # Find line of intersection between two planes
L = np.cross(N1, N2) L = np.cross(N1, N2)
# Find two intersection points # Find two intersection points
X1 = L / np.sqrt(L[0]**2 + L[1]**2 + L[2]**2) X1 = L / np.sqrt(L[0] ** 2 + L[1] ** 2 + L[2] ** 2)
X2 = -X1 X2 = -X1
def mag(q): def mag(q):
return np.sqrt(np.vdot(q, q)) return np.sqrt(np.vdot(q, q))
dist1 = mag(X1 - plane_a[0]) dist1 = mag(X1 - plane_a[0])
dist2 = mag(X2 - plane_a[0]) dist2 = mag(X2 - plane_a[0])
# return the (lon_lat pair of the closer intersection) # return the (lon_lat pair of the closer intersection)
...@@ -243,33 +278,29 @@ def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=10000 ...@@ -243,33 +278,29 @@ def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance=10000
if abs(check_bearing - doa_a) < 5: if abs(check_bearing - doa_a) < 5:
km = v.inverse([lat_a, lon_a], [i_lat, i_long]) km = v.inverse([lat_a, lon_a], [i_lat, i_long])
if km[0] < max_distance: if km[0] < max_distance:
return (i_lat, i_long) return i_lat, i_long
else: else:
return None return None
####################################################################### def do_dbscan(x, epsilon, min_samp):
# We start this in it's own process do it doesn't eat all of your RAM. """ We start this in it's own process do it doesn't eat all of your RAM.
# This becomes noticable at over 10k intersections. This becomes noticable at over 10k intersections."""
#######################################################################
def do_dbscan(X, epsilon, minsamp):
DBSCAN_WAIT_Q.put(True) DBSCAN_WAIT_Q.put(True)
db = DBSCAN(eps=epsilon, min_samples=minsamp).fit(X) db = DBSCAN(eps=epsilon, min_samples=min_samp).fit(x)
DBSCAN_Q.put(db.labels_) DBSCAN_Q.put(db.labels_)
if not DBSCAN_WAIT_Q.empty(): if not DBSCAN_WAIT_Q.empty():
DBSCAN_WAIT_Q.get() DBSCAN_WAIT_Q.get()
#################################### def autoeps_calc(xs):
# Autocalculate the best eps value. """ Auto calculate the best eps value. """
####################################
def autoeps_calc(X):
# only use a sample of the data to speed up calculation. # only use a sample of the data to speed up calculation.
X = X[:min(2000, len(X)):2] xs = xs[:min(2000, len(xs)):2]
min_distances = [] min_distances = []
for x in X: for x in xs:
distances = [] distances = []
for y in X: for y in xs:
# calculate euclidian distance # calculate euclidian distance
distance = math.sqrt(sum([(a - b) ** 2 for a, b in zip(x, y)])) distance = math.sqrt(sum([(a - b) ** 2 for a, b in zip(x, y)]))
if distance > 0: if distance > 0:
...@@ -292,11 +323,10 @@ def autoeps_calc(X): ...@@ -292,11 +323,10 @@ def autoeps_calc(X):
return 0 return 0
############################################### # noinspection PyShadowingNames
# Computes DBSCAN Alorithm is applicable,
# finds the mean of a cluster of intersections.
###############################################
def process_data(database_name, epsilon, min_samp): def process_data(database_name, epsilon, min_samp):
""" Computes DBSCAN Alorithm is applicable,
finds the mean of a cluster of intersections."""
n_std = 3.0 n_std = 3.0
intersect_list = [] intersect_list = []
likely_location = [] likely_location = []
...@@ -311,7 +341,7 @@ def process_data(database_name, epsilon, min_samp): ...@@ -311,7 +341,7 @@ def process_data(database_name, epsilon, min_samp):
aoi_list.append(-1) aoi_list.append(-1)
for aoi in aoi_list: for aoi in aoi_list:
print(f"Checking AOI {aoi}.") logging.debug(f"Checking AOI {aoi}.")
curs.execute('''SELECT longitude, latitude, time FROM intersects curs.execute('''SELECT longitude, latitude, time FROM intersects
WHERE aoi_id=? ORDER BY confidence DESC LIMIT 25000''', [aoi]) WHERE aoi_id=? ORDER BY confidence DESC LIMIT 25000''', [aoi])
intersect_array = np.array(curs.fetchall()) intersect_array = np.array(curs.fetchall())
...@@ -330,7 +360,7 @@ def process_data(database_name, epsilon, min_samp): ...@@ -330,7 +360,7 @@ def process_data(database_name, epsilon, min_samp):
if epsilon == "auto": if epsilon == "auto":
epsilon = autoeps_calc(X) epsilon = autoeps_calc(X)
print(f"min_samp: {min_samp}, eps: {epsilon}") logger.debug(f"min_samp: {min_samp}, eps: {epsilon}")
else: else:
try: try:
epsilon = float(epsilon) epsilon = float(epsilon)
...@@ -339,36 +369,37 @@ def process_data(database_name, epsilon, min_samp): ...@@ -339,36 +369,37 @@ def process_data(database_name, epsilon, min_samp):
# size_x = sys.getsizeof(X)/1024 # size_x = sys.getsizeof(X)/1024
# print(f"The dataset is {size_x} kilobytes") # print(f"The dataset is {size_x} kilobytes")
print(f"Computing Clusters from {n_points} intersections.") logger.info(f"Computing Clusters "
f"from {n_points} intersections.")
while not DBSCAN_WAIT_Q.empty(): while not DBSCAN_WAIT_Q.empty():
print("Waiting for my turn...") logger.debug("Waiting for my turn...")
time.sleep(1) time.sleep(1)
starttime = time.time() starttime = time.time()
db = Process(target=do_dbscan, args=(X, epsilon, min_samp)) db = Process(target=do_dbscan, args=(X, epsilon, min_samp))
db.daemon = True db.daemon = True
db.start() db.start()
# noinspection PyBroadException
try: try:
labels = DBSCAN_Q.get(timeout=10) labels = DBSCAN_Q.get(timeout=10)
db.join() db.join()
except: except Exception:
print("DBSCAN took took long, terminated.") logger.warning("DBSCAN took took long, terminated.")
if not DBSCAN_WAIT_Q.empty(): if not DBSCAN_WAIT_Q.empty():
DBSCAN_WAIT_Q.get() DBSCAN_WAIT_Q.get()
db.terminate() db.terminate()
return likely_location, intersect_list, ellipsedata return likely_location, intersect_list, ellipsedata
stoptime = time.time() stoptime = time.time()
print( logger.info(f"DBSCAN took {stoptime - starttime}"
f"DBSCAN took {stoptime - starttime} seconds to compute the clusters.") f"seconds to compute the clusters.")
intersect_array = np.column_stack((intersect_array, labels)) intersect_array = np.column_stack((intersect_array, labels))
# Number of clusters in labels, ignoring noise if present. # Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
n_noise_ = list(labels).count(-1) n_noise_ = list(labels).count(-1)
clear(debugging) logger.debug('Number of clusters: %d' % n_clusters_)
print('Number of clusters: %d' % n_clusters_) logger.debug('Outliers Removed: %d' % n_noise_)
print('Outliers Removed: %d' % n_noise_)
for x in range(n_clusters_): for x in range(n_clusters_):
cluster = np.array([]).reshape(0, 3) cluster = np.array([]).reshape(0, 3)
...@@ -376,34 +407,39 @@ def process_data(database_name, epsilon, min_samp): ...@@ -376,34 +407,39 @@ def process_data(database_name, epsilon, min_samp):
if intersect_array[y][-1] == x: if intersect_array[y][-1] == x:
cluster = np.concatenate( cluster = np.concatenate(
(cluster, [intersect_array[y][0:-1]]), axis=0) (cluster, [intersect_array[y][0:-1]]), axis=0)
# weighted_location.append(np.average(cluster[:,0:2], weights=cluster[:,2], axis=0).tolist()) # weighted_location.append(np.average(cluster[:,0:2],
# weights=cluster[:,2],
# axis=0).tolist())
clustermean = np.mean(cluster[:, 0:2], axis=0) clustermean = np.mean(cluster[:, 0:2], axis=0)
likely_location.append(clustermean.tolist()) likely_location.append(clustermean.tolist())
cov = np.cov(cluster[:, 0], cluster[:, 1]) cov = np.cov(cluster[:, 0], cluster[:, 1])
a = cov[0, 0] a = cov[0, 0]
b = cov[0, 1] b = cov[0, 1]
c = cov[1, 1] c = cov[1, 1]
if (a == 0.0 or b == 0.0 or c == 0.0): if a == 0.0 or b == 0.0 or c == 0.0:
if debugging: if debugging:
print(f"A: {a} B: {b} C: {c}") logger.debug(f"A: {a} B: {b} C: {c}")
print("Unable to resolve ellipse.") logger.debug("Unable to resolve ellipse.")
break break
lam1 = a + c / 2 + np.sqrt((a - c / 2)**2 + b**2) lam1 = a + c / 2 + np.sqrt((a - c / 2) ** 2 + b ** 2)
# lam2 = a+c/2 - np.sqrt((a-c/2)**2 + b**2) # lam2 = a+c/2 - np.sqrt((a-c/2)**2 + b**2)
# print([lam1, lam2, a, c]) # print([lam1, lam2, a, c])
pearson = b / np.sqrt(a * c) pearson = b / np.sqrt(a * c)
if (1 + pearson < 0.0 or 1 - pearson < 0.0): if 1 + pearson < 0.0 or 1 - pearson < 0.0:
if debugging: if debugging:
print(f"Pearson Value: {pearson}") logger.debug(f"Pearson Value: {pearson}")
print("Unable to resolve ellipse.") logger.debug("Unable to resolve ellipse.")
break break
print(f"A: {a} B: {b} C: {c} pearson: {pearson}") logger.debug(f"A: {a} B: {b} C: {c} pearson: {pearson}")
ell_radius_x = np.sqrt(1 + pearson) * np.sqrt(a) * n_std ell_radius_x = np.sqrt(1 + pearson) * np.sqrt(a) * n_std
ell_radius_y = np.sqrt(1 - pearson) * np.sqrt(c) * n_std ell_radius_y = np.sqrt(1 - pearson) * np.sqrt(c) * n_std
axis_x = v.inverse(clustermean.tolist()[ axis_x = v.inverse(clustermean.tolist()[
::-1], (ell_radius_x + clustermean[1], clustermean[0]))[0] ::-1], (ell_radius_x + clustermean[1],
clustermean[0]))[0]
axis_y = v.inverse(clustermean.tolist()[ axis_y = v.inverse(clustermean.tolist()[
::-1], (clustermean[1], ell_radius_y + clustermean[0]))[0] ::-1], (clustermean[1],
ell_radius_y + clustermean[0]))[
0]
if b == 0 and a >= c: if b == 0 and a >= c:
rotation = 0 rotation = 0
elif b == 0 and a < c: elif b == 0 and a < c:
...@@ -415,7 +451,7 @@ def process_data(database_name, epsilon, min_samp): ...@@ -415,7 +451,7 @@ def process_data(database_name, epsilon, min_samp):
[axis_x, axis_y, rotation, *clustermean.tolist()]) [axis_x, axis_y, rotation, *clustermean.tolist()])
for x in likely_location: for x in likely_location:
print(x[::-1]) logger.debug(x[::-1])
for x in intersect_array: for x in intersect_array:
try: try:
...@@ -425,16 +461,14 @@ def process_data(database_name, epsilon, min_samp): ...@@ -425,16 +461,14 @@ def process_data(database_name, epsilon, min_samp):
intersect_list.append(x.tolist()) intersect_list.append(x.tolist())
else: else:
print(f"No Intersections in AOI {aoi}.") logger.debug(f"No Intersections in AOI {aoi}.")
conn.close() conn.close()
return likely_location, intersect_list, ellipsedata return likely_location, intersect_list, ellipsedata
#######################################################################
# Checks interesections stored in the database against a lat/lon/radius
# and removes items inside exclusion areas.
#######################################################################
def purge_database(type, lat, lon, radius): def purge_database(type, lat, lon, radius):
""" Checks interesections stored in the database against a lat/lon/radius
and removes items inside exclusion areas."""
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
c = conn.cursor() c = conn.cursor()
c.execute("SELECT latitude, longitude, id FROM intersects") c.execute("SELECT latitude, longitude, id FROM intersects")
...@@ -453,7 +487,7 @@ def purge_database(type, lat, lon, radius): ...@@ -453,7 +487,7 @@ def purge_database(type, lat, lon, radius):
DATABASE_EDIT_Q.put((command, delete_these, False)) DATABASE_EDIT_Q.put((command, delete_these, False))
# DATABASE_RETURN.get(timeout=1) # DATABASE_RETURN.get(timeout=1)
DATABASE_EDIT_Q.put(("done", None, False)) DATABASE_EDIT_Q.put(("done", None, False))
print(f"I purged {purge_count} intersects.") logger.info(f"I purged {purge_count} intersects.")
############################################### ###############################################
...@@ -525,7 +559,9 @@ def run_aoi_rules(): ...@@ -525,7 +559,9 @@ def run_aoi_rules():
DATABASE_EDIT_Q.put(("done", None, False)) DATABASE_EDIT_Q.put(("done", None, False))
stoptime = time.time() stoptime = time.time()
print(f"Purged {purged} intersections and sorted {sorted} intersections into {n_aoi} AOIs in {stoptime - starttime} seconds.") logger.info(f"Purged {purged} intersections and sorted {sorted}"
f"intersections into {n_aoi} AOIs"
f"in {stoptime - starttime} seconds.")
return "OK" return "OK"
...@@ -538,25 +574,27 @@ def write_geojson(best_point, all_the_points): ...@@ -538,25 +574,27 @@ def write_geojson(best_point, all_the_points):
"marker-color": "#00FF00"} "marker-color": "#00FF00"}
if all_the_points is not None: if all_the_points is not None:
all_the_points = Feature( all_the_points = Feature(
properties=all_pt_style, geometry=MultiPoint(tuple(all_the_points))) properties=all_pt_style,
geometry=MultiPoint(tuple(all_the_points)))
with open(geofile, "w") as file1: with open(geofile, "w") as file1:
if best_point is not None: if best_point is not None:
reversed_best_point = [] reversed_best_point = []
for x in best_point: for x in best_point:
reversed_best_point.append(x) reversed_best_point.append(x)
best_point = Feature(properties=best_pt_style, geometry=MultiPoint( best_point = Feature(properties=best_pt_style,
tuple(reversed_best_point))) geometry=MultiPoint(
tuple(reversed_best_point)))
file1.write(str(FeatureCollection( file1.write(str(FeatureCollection(
[best_point, all_the_points]))) [best_point, all_the_points])))
else: else:
file1.write(str(FeatureCollection([all_the_points]))) file1.write(str(FeatureCollection([all_the_points])))
print(f"Wrote file {geofile}") logger.debug(f"Wrote file {geofile}")
############################################### # noinspection PyPep8Naming
# Writes output.czml used by the WebUI def write_czml(best_point, all_the_points, ellipsedata,
############################################### plotallintersects, eps):
def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps): """ Writes output.czml used by the WebUI """
point_properties = { point_properties = {
"pixelSize": 5.0, "pixelSize": 5.0,
"heightReference": "CLAMP_TO_GROUND", "heightReference": "CLAMP_TO_GROUND",
...@@ -594,22 +632,28 @@ def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps): ...@@ -594,22 +632,28 @@ def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps):
all_the_points = np.column_stack((all_the_points, scaled_time)) all_the_points = np.column_stack((all_the_points, scaled_time))
for x in all_the_points: for x in all_the_points:
# rgb = hsvtorgb(x[-1]/3, 0.9, 0.9) # rgb = hsvtorgb(x[-1]/3, 0.9, 0.9)
rgb = map(lambda x: int(x * 255), hsv_to_rgb(x[-1] / 3, 0.9, 0.9)) rgb = map(lambda c: int(c * 255), hsv_to_rgb(x[-1] / 3, 0.9, 0.9))
color_property = {"color": {"rgba": [*rgb, 255]}} color_property = {"color": {"rgba": [*rgb, 255]}}
all_point_packets.append(Packet(id=str(x[1]) + ", " + str(x[0]), all_point_packets.append(Packet(id=str(x[1]) + ", " + str(x[0]),
point={**point_properties, point={**point_properties,
**color_property}, **color_property},
position={ position={
"cartographicDegrees": [x[0], x[1], 0]}, "cartographicDegrees": [x[0],
)) x[1],
0]},
))
if len(best_point) > 0: if len(best_point) > 0:
for x in best_point: for x in best_point:
gmaps_url = f"https://www.google.com/maps/dir/?api=1&destination={x[1]},+{x[0]}&travelmode=driving" gmaps_url = (f"https://www.google.com/maps/dir/?api=1&"
best_point_packets.append(Packet(id=str(x[1]) + ", " + str(x[0]), f"destination={x[1]},+{x[0]}&travelmode=driving")
point=best_point_properties, best_point_packets.append(
description=f"<a href='{gmaps_url}' target='_blank'>Google Maps Directions</a>", Packet(id=str(x[1]) + ", " + str(x[0]),
position={"cartographicDegrees": [x[0], x[1], 0]})) point=best_point_properties,
description=(f"<a href='{gmaps_url}' target='_blank'>"
f"Google Maps Directions</a>"),
position={"cartographicDegrees": [x[0], x[1], 0]})
)
if len(ellipsedata) > 0: if len(ellipsedata) > 0:
for x in ellipsedata: for x in ellipsedata:
...@@ -621,31 +665,39 @@ def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps): ...@@ -621,31 +665,39 @@ def write_czml(best_point, all_the_points, ellipsedata, plotallintersects, eps):
rotation = 2 * np.pi - x[2] rotation = 2 * np.pi - x[2]
rotation += np.pi / 2 rotation += np.pi / 2
# print(f"{x[2]} Inverted to: {rotation}") # print(f"{x[2]} Inverted to: {rotation}")
# print(f"SemiMajor: {semiMajorAxis}, Semiminor: {semiMinorAxis}") # print(f"SemiMajor: {semiMajorAxis},
# f"Semiminor: {semiMinorAxis}")
# print(f"{x[4], x[3]} is inveted") # print(f"{x[4], x[3]} is inveted")
else: else:
rotation = x[2] rotation = x[2]
semiMajorAxis = x[1] semiMajorAxis = x[1]
semiMinorAxis = x[0] semiMinorAxis = x[0]
# print(f"Not inverted: {rotation}") # print(f"Not inverted: {rotation}")
# print(f"SemiMajor: {semiMajorAxis}, Semiminor: {semiMinorAxis}") # print(f"SemiMajor: {semiMajorAxis},"
# f"Semiminor: {semiMinorAxis}")
# print(f"{x[4], x[3]} is NOT inveted") # print(f"{x[4], x[3]} is NOT inveted")
ellipse_info = {"semiMajorAxis": semiMajorAxis, ellipse_info = {"semiMajorAxis": semiMajorAxis,
"semiMinorAxis": semiMinorAxis, "rotation": rotation} "semiMinorAxis": semiMinorAxis,
"rotation": rotation}
ellipse_packets.append(Packet(id=str(x[4]) + ", " + str(x[3]), ellipse_packets.append(Packet(id=str(x[4]) + ", " + str(x[3]),
ellipse={ ellipse={
**ellipse_properties, **ellipse_info}, **ellipse_properties,
position={"cartographicDegrees": [x[3], x[4], 0]})) **ellipse_info},
position={
"cartographicDegrees": [x[3],
x[4],
0]}))
document_list = [top, ]
document_list.extend(best_point_packets)
document_list.extend(all_point_packets)
document_list.extend(ellipse_packets)
return Document(document_list).dumps(separators=(',', ':'))
return Document([top] + best_point_packets + all_point_packets + ellipse_packets).dumps(separators=(',', ':'))
###############################################
# Writes receivers.czml used by the WebUI
###############################################
@get('/receivers.czml') @get('/receivers.czml')
def write_rx_czml(): def write_rx_czml():
""" Writes receivers.czml used by the WebUI """
response.set_header( response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0', 'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0',
) )
...@@ -671,114 +723,140 @@ def write_rx_czml(): ...@@ -671,114 +723,140 @@ def write_rx_czml():
"height": 48, "height": 48,
"width": 48, "width": 48,
} }
while not ms.rx_busy: for index, x in enumerate(receivers):
for index, x in enumerate(receivers): if x.isActive and ms.receiving:
if x.isActive and ms.receiving: if x.confidence > min_conf and x.power > min_power:
if (x.confidence > min_conf and x.power > min_power): lob_color = green
lob_color = green elif x.confidence <= min_conf and x.power > min_power:
elif (x.confidence <= min_conf and x.power > min_power): lob_color = orange
lob_color = orange
else:
lob_color = red
lob_start_lat = x.latitude
lob_start_lon = x.longitude
lob_stop_lat, lob_stop_lon = v.direct(
lob_start_lat, lob_start_lon, x.doa, x.lob_length())
lob_packets.append(Packet(id=f"LOB-{x.station_id}-{index}",
polyline=Polyline(
material=Material(polylineOutline=PolylineOutlineMaterial(
color=Color(
rgba=lob_color),
outlineColor=Color(
rgba=[0, 0, 0, 255]),
outlineWidth=2
)),
clampToGround=True,
width=5,
positions=Position(cartographicDegrees=[
lob_start_lon, lob_start_lat, height, lob_stop_lon, lob_stop_lat, height])
)))
heading_start_lat = x.latitude
heading_start_lon = x.longitude
heading_stop_lat, heading_stop_lon = v.direct(
heading_start_lat, heading_start_lon, x.heading, heading_d)
lob_packets.append(Packet(id=f"HEADING-{x.station_id}-{index}",
polyline=Polyline(
material=PolylineMaterial(
polylineDash = PolylineDashMaterial(color=Color(
rgba=gray),
gapColor=Color(
rgba=[0, 0, 0, 0])
)),
clampToGround=True,
width=2,
positions=Position(cartographicDegrees=[
heading_start_lon, heading_start_lat, height, heading_stop_lon, heading_stop_lat, height])
)))
else:
lob_packets = []
if x.isMobile is True:
rx_icon = {"image": {"uri": "/static/flipped_car.svg"}}
# if x.heading > 0 or x.heading < 180:
# rx_icon = {"image":{"uri":"/static/flipped_car.svg"}, "rotation":math.radians(360 - x.heading + 90)}
# elif x.heading < 0 or x.heading > 180:
# rx_icon = {"image":{"uri":"/static/car.svg"}, "rotation":math.radians(360 - x.heading - 90)}
else: else:
rx_icon = {"image": {"uri": "/static/tower.svg"}} lob_color = red
receiver_point_packets.append(Packet(id=f"{x.station_id}-{index}", lob_start_lat = x.latitude
billboard={ lob_start_lon = x.longitude
**rx_properties, **rx_icon}, lob_stop_lat, lob_stop_lon = v.direct(
position={"cartographicDegrees": [x.longitude, x.latitude, 15]})) lob_start_lat, lob_start_lon, x.doa, x.lob_length()
)
response_data = Document([top] + receiver_point_packets + lob_packets).dumps(separators=(',', ':')) lob_packets.append(
# 11.01.2023::s1z: I know this is ugly and stupid, Packet(
# but we're not gonna rewrite the project anyways, id=f"LOB-{x.station_id}-{index}",
# so, whatever... polyline=Polyline(
globals()["cached_receivers"] = response_data material=Material(
return response_data polylineOutline=PolylineOutlineMaterial(
return globals()["cached_receivers"] color=Color(rgba=lob_color),
outlineColor=Color(rgba=[0, 0, 0, 255]),
outlineWidth=2
)
),
clampToGround=True,
width=5,
positions=Position(
cartographicDegrees=[
lob_start_lon,
lob_start_lat,
height,
lob_stop_lon,
lob_stop_lat,
height
]
)
)
)
)
heading_start_lat = x.latitude
heading_start_lon = x.longitude
heading_stop_lat, heading_stop_lon = v.direct(
heading_start_lat, heading_start_lon, x.heading, heading_d)
lob_packets.append(
Packet(
id=f"HEADING-{x.station_id}-{index}",
polyline=Polyline(
material=PolylineMaterial(
polylineDash=PolylineDashMaterial(
color=Color(rgba=gray),
gapColor=Color(rgba=[0, 0, 0, 0])
)
),
clampToGround=True,
width=2,
positions=Position(
cartographicDegrees=[
heading_start_lon,
heading_start_lat,
height,
heading_stop_lon,
heading_stop_lat,
height]
)
)
)
)
else:
lob_packets = []
if x.isMobile is True:
rx_icon = {"image": {"uri": "/static/flipped_car.svg"}}
# if x.heading > 0 or x.heading < 180:
# rx_icon = {"image":{"uri":"/static/flipped_car.svg"},
# "rotation":math.radians(360 - x.heading + 90)}
# elif x.heading < 0 or x.heading > 180:
# rx_icon = {"image":{"uri":"/static/car.svg"},
# "rotation":math.radians(360 - x.heading - 90)}
else:
rx_icon = {"image": {"uri": "/static/tower.svg"}}
receiver_point_packets.append(
Packet(id=f"{x.station_id}-{index}",
billboard={**rx_properties, **rx_icon},
position={"cartographicDegrees": [x.longitude,
x.latitude, 15]})
)
document_list = [top, ]
document_list.extend(receiver_point_packets)
document_list.extend(lob_packets)
response_data = Document(document_list).dumps(separators=(',', ':'))
shared_dict["response"] = response_data
return response_data
###############################################
# Writes aoi.czml used by the WebUI
###############################################
@get("/aoi.czml") @get("/aoi.czml")
def wr_aoi_czml(): def wr_aoi_czml():
""" Writes aoi.czml used by the WebUI """
response.set_header( response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0') 'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
aoi_packets = [] aoi_packets = []
top = Preamble(name="AOIs") top = Preamble(name="AOIs")
area_of_interest_properties = { area_of_interest_properties = {
"granularity": 0.008722222, "granularity": 0.008722222,
"height": 0, "height": 0,
# "zIndex": 1, # "zIndex": 1,
"material": { "material": {
"solidColor": { "solidColor": {
"color": { "color": {
"rgba": [0, 0, 255, 25] "rgba": [0, 0, 255, 25]
} }
} }
}, },
"outline": True, "outline": True,
"outlineWidth": 2, "outlineWidth": 2,
"outlineColor": {"rgba": [53, 184, 240, 255], }, "outlineColor": {
}, "rgba": [53, 184, 240, 255], },
},
exclusion_area_properties = { exclusion_area_properties = {
"granularity": 0.008722222, "granularity": 0.008722222,
"height": 0, "height": 0,
# "zIndex": 0, # "zIndex": 0,
"material": { "material": {
"solidColor": { "solidColor": {
"color": { "color": {
"rgba": [242, 10, 0, 25] "rgba": [242, 10, 0, 25]
} }
} }
}, },
"outline": True, "outline": True,
"outlineWidth": 2, "outlineWidth": 2,
"outlineColor": {"rgba": [224, 142, 0, 255], }, "outlineColor": {
}, "rgba": [224, 142, 0, 255], },
},
for x in fetch_aoi_data(): for x in fetch_aoi_data():
aoi = { aoi = {
...@@ -796,65 +874,54 @@ def wr_aoi_czml(): ...@@ -796,65 +874,54 @@ def wr_aoi_czml():
"semiMinorAxis": aoi['radius'], "rotation": 0} "semiMinorAxis": aoi['radius'], "rotation": 0}
aoi_packets.append(Packet(id=aoi['aoi_type'] + str(aoi['uid']), aoi_packets.append(Packet(id=aoi['aoi_type'] + str(aoi['uid']),
ellipse={**aoi_properties, **aoi_info}, ellipse={**aoi_properties, **aoi_info},
position={"cartographicDegrees": [aoi['longitude'], aoi['latitude'], 0]})) position={
"cartographicDegrees": [aoi['longitude'],
aoi['latitude'],
0]}))
return Document([top] + aoi_packets).dumps(separators=(',', ':')) return Document([top] + aoi_packets).dumps(separators=(',', ':'))
###############################################
# Clears the screen if debugging is off.
###############################################
def clear(debugging):
if not debugging:
# for windows
if name == 'nt':
_ = system('cls')
# for mac and linux(here, os.name is 'posix')
else:
_ = system('clear')
###############################################
# Serves static files such as CSS and JS to the
# WebUI
###############################################
@route('/static/<filepath:path>', name='static') @route('/static/<filepath:path>', name='static')
def server_static(filepath): def server_static(filepath):
""" Serves static files such as CSS and JS to the WebUI """
response = static_file(filepath, root='./static') response = static_file(filepath, root='./static')
response.set_header( response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0') 'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
return response return response
###############################################
# Loads the main page of the WebUI
# http://[ip]:[port]/
###############################################
@get('/') @get('/')
@get('/index') @get('/index')
@get('/cesium') @get('/cesium')
def cesium(): def cesium():
""" Loads the main page of the WebUI http://[ip]:[port]/ """
response.set_header( response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0') 'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
return template('cesium.tpl', return template(
{'access_token': access_token, 'cesium.tpl',
'epsilon': ms.eps, {
'minpower': ms.min_power, 'access_token': access_token,
'minconf': ms.min_conf, 'epsilon': ms.eps,
'minpoints': ms.min_samp, 'minpower': ms.min_power,
'rx_state': "checked" if ms.receiving is True else "", 'minconf': ms.min_conf,
'intersect_state': "checked" if ms.plotintersects is True else "", 'minpoints': ms.min_samp,
'receivers': receivers}) 'rx_state': "checked" if ms.receiving is True else "",
'intersect_state': "checked" if ms.plotintersects is True else "",
'receivers': receivers
}
)
############################################### # noinspection SpellCheckingInspection
# GET Request to update parameters from the
# UI sliders. Not meant to be user facing.
###############################################
@get('/update') @get('/update')
def update_cesium(): def update_cesium():
""" GET Request to update parameters from the
UI sliders. Not meant to be user facing. """
# eps = float(request.query.eps) if request.query.eps else ms.eps # eps = float(request.query.eps) if request.query.eps else ms.eps
# min_samp = float(request.query.minpts) if request.query.minpts else ms.min_samp # min_samp = (float(request.query.minpts)
# if request.query.minpts
# else ms.min_samp)
ms.min_conf = float( ms.min_conf = float(
request.query.minconf) if request.query.minconf else ms.min_conf request.query.minconf) if request.query.minconf else ms.min_conf
ms.min_power = float( ms.min_power = float(
...@@ -873,17 +940,15 @@ def update_cesium(): ...@@ -873,17 +940,15 @@ def update_cesium():
return "OK" return "OK"
############################################### # noinspection PyShadowingNames
# Returns a JSON file to the WebUI with
# information to fill in the RX cards.
###############################################
@get('/rx_params') @get('/rx_params')
def rx_params(): def rx_params():
""" Returns a JSON file to the WebUI
with information to fill in the RX cards. """
all_rx = {'receivers': {}} all_rx = {'receivers': {}}
rx_properties = [] rx_properties = []
for index, x in enumerate(receivers): for index, x in enumerate(receivers):
x.update() # x.update() # this line blocks web execution, should be async!
rx = x.receiver_dict() rx = x.receiver_dict()
rx['uid'] = index rx['uid'] = index
rx_properties.append(rx) rx_properties.append(rx)
...@@ -892,12 +957,11 @@ def rx_params(): ...@@ -892,12 +957,11 @@ def rx_params():
return json.dumps(all_rx) return json.dumps(all_rx)
############################################### # noinspection SpellCheckingInspection
# Returns a CZML file that contains intersect
# and ellipse information for Cesium.
###############################################
@get('/output.czml') @get('/output.czml')
def tx_czml_out(): def tx_czml_out():
""" Returns a CZML file that contains intersect
and ellipse information for Cesium. """
eps = request.query.eps if request.query.eps else str(ms.eps) eps = request.query.eps if request.query.eps else str(ms.eps)
min_samp = request.query.minpts if request.query.minpts else str( min_samp = request.query.minpts if request.query.minpts else str(
ms.min_samp) ms.min_samp)
...@@ -909,20 +973,22 @@ def tx_czml_out(): ...@@ -909,20 +973,22 @@ def tx_czml_out():
plotallintersects = ms.plotintersects plotallintersects = ms.plotintersects
response.set_header( response.set_header(
'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0') 'Cache-Control', 'no-cache, no-store, must-revalidate, max-age=0')
output = write_czml(*process_data(database_name, eps, output = write_czml(*process_data(database_name, eps, min_samp),
min_samp), plotallintersects, eps) plotallintersects, eps)
return str(output) return str(output)
############################################### # noinspection PyPep8Naming,SqlDialectInspection,PyShadowingNames
# PUT request to update receiver variables
# from the WebUI
###############################################
@put('/rx_params/<action>') @put('/rx_params/<action>')
def update_rx(action): def update_rx(action):
""" PUT request to update receiver variables from the WebUI """
data = json.load(request.body) data = json.load(request.body)
if action == "new": if action == "new":
receiver_url = data['station_url'].replace('\n', '') receiver_url = data['station_url'].strip()
if any(x.station_url == receiver_url for x in receivers):
logger.warning(f"The station with the url '{receiver_url}' "
"already exists!")
return redirect('/rx_params')
add_receiver(receiver_url) add_receiver(receiver_url)
elif action == "del": elif action == "del":
index = int(data['uid']) index = int(data['uid'])
...@@ -946,16 +1012,15 @@ def update_rx(action): ...@@ -946,16 +1012,15 @@ def update_rx(action):
receivers[action].update() receivers[action].update()
update_rx_table() update_rx_table()
except IndexError: except IndexError:
print("I got some bad data. Doing nothing out of spite.") logger.error("I got some bad data. Doing nothing out of spite.")
return redirect('/rx_params') return redirect('/rx_params')
############################################### # noinspection SpellCheckingInspection
# Returns a JSON file to the WebUI with
# information to fill in the AOI cards.
###############################################
@get('/interest_areas') @get('/interest_areas')
def load_interest_areas(): def load_interest_areas():
""" Returns a JSON file to the WebUI with
information to fill in the AOI cards. """
all_aoi = {'aois': {}} all_aoi = {'aois': {}}
aoi_properties = [] aoi_properties = []
for x in fetch_aoi_data(): for x in fetch_aoi_data():
...@@ -972,13 +1037,12 @@ def load_interest_areas(): ...@@ -972,13 +1037,12 @@ def load_interest_areas():
return json.dumps(all_aoi) return json.dumps(all_aoi)
########################################## # noinspection SqlNoDataSourceInspection,SqlDialectInspection
# PUT request to add new AOI to DB
##########################################
@put('/interest_areas/<action>') @put('/interest_areas/<action>')
def handle_interest_areas(action): def handle_interest_areas(action):
""" PUT request to add new AOI to DB """
data = json.load(request.body) data = json.load(request.body)
if action == "new" and not "" in data.values(): if action == "new" and "" not in data.values():
aoi_type = data['aoi_type'] aoi_type = data['aoi_type']
lat = data['latitude'] lat = data['latitude']
lon = data['longitude'] lon = data['longitude']
...@@ -996,34 +1060,65 @@ def handle_interest_areas(action): ...@@ -996,34 +1060,65 @@ def handle_interest_areas(action):
elif action == "purge": elif action == "purge":
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
c = conn.cursor() c = conn.cursor()
c.execute("SELECT aoi_type, latitude, longitude, radius FROM interest_areas WHERE uid=?", [ c.execute("SELECT aoi_type, latitude, longitude, radius"
data['uid']]) "FROM interest_areas WHERE uid=?", [data['uid']])
properties = c.fetchone() properties = c.fetchone()
conn.close() conn.close()
purge_database(*properties) purge_database(*properties)
############################################### class BottleWebServer(threading.Thread):
# Starts the Bottle webserver. """ Database Writer thread """
###############################################
def start_server(ipaddr="127.0.0.1", port=8080):
try:
run(host=ipaddr, port=port, quiet=True,
server=GeventWebSocketServer, debug=True)
except OSError:
print(f"Port {port} seems to be in use. Please select another port or " +
"check if another instance of DFA is already running.")
debugging = True
finish()
def __init__(self, interface="127.0.0.1", port=8080):
super().__init__(name=self.__class__.__name__, daemon=True)
self.interface = interface
self.port = port
self._is_running = False
self._is_running_lock = threading.Lock()
############################################### @property
# Captures DOA data and computes intersections def is_running(self):
# if the receiver is enabled. Writes the """ Get _is_running safely """
# intersections to the database. with self._is_running_lock:
############################################### return self._is_running
@is_running.setter
def is_running(self, value: bool):
""" set _is_running safely """
with self._is_running_lock:
self._is_running = value
def stop(self):
""" Stop the Thread """
self.is_running = False
def start(self):
""" Start thread """
self.is_running = True
super().start()
def run(self):
""" Starts the Bottle web server. """
# noinspection PyUnresolvedReferences
from bottle.ext.websocket import GeventWebSocketServer
try:
run(host=self.interface, port=self.port, quiet=True,
server=GeventWebSocketServer, debug=True)
except OSError:
logger.error(f"Port {self.port} seems to be in use."
f"Please select another port or "
f"check if another instance of DFA is "
f"already running.")
sys.exit(1)
# noinspection PyShadowingNames,SqlDialectInspection
def run_receiver(receivers): def run_receiver(receivers):
clear(debugging) """ Captures DOA data and computes intersections
if the receiver is enabled.
Writes the intersections to the database."""
dots = 0 dots = 0
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
...@@ -1031,8 +1126,7 @@ def run_receiver(receivers): ...@@ -1031,8 +1126,7 @@ def run_receiver(receivers):
while ms.receiving: while ms.receiving:
if not debugging: if not debugging:
print("Receiving" + dots * '.') logger.info("Receiving" + dots * '.')
print("Press Control+C to process data and exit.")
# Main loop to compute intersections between multiple receivers # Main loop to compute intersections between multiple receivers
intersect_list = np.array([]).reshape(0, 3) intersect_list = np.array([]).reshape(0, 3)
...@@ -1045,32 +1139,35 @@ def run_receiver(receivers): ...@@ -1045,32 +1139,35 @@ def run_receiver(receivers):
# We have to change this to the queue for each receiver. # We have to change this to the queue for each receiver.
rx.update() rx.update()
except IOError: except IOError:
print("Problem connecting to receiver.") logger.error("Problem connecting to receiver.")
rx.d_2_last_intersection = [] rx.d_2_last_intersection = []
time.sleep(0.1) time.sleep(0.1)
# for x in range(len(receivers)):
# for y in range(x):
# if x != y:
receivers_len = len(receivers) receivers_len = len(receivers)
for x in range(receivers_len): for x in range(receivers_len):
y_receiver_id = x + 1 y_receiver_id = x + 1
for y in range(y_receiver_id, receivers_len): for y in range(y_receiver_id, receivers_len):
if (receivers[x].confidence >= ms.min_conf and if (receivers[x].confidence >= ms.min_conf and
receivers[y].confidence >= ms.min_conf and receivers[y].confidence >= ms.min_conf and
receivers[x].power >= ms.min_power and receivers[x].power >= ms.min_power and
receivers[y].power >= ms.min_power and receivers[y].power >= ms.min_power and
abs(receivers[x].doa_time - receivers[y].doa_time) <= max_age and abs(receivers[x].doa_time - receivers[
y].doa_time) <= max_age and
receivers[x].frequency == receivers[y].frequency): receivers[x].frequency == receivers[y].frequency):
intersection = plot_intersects(receivers[x].latitude, receivers[x].longitude, intersection = plot_intersects(receivers[x].latitude,
receivers[x].doa, receivers[y].latitude, receivers[y].longitude, receivers[y].doa) receivers[x].longitude,
receivers[x].doa,
receivers[y].latitude,
receivers[y].longitude,
receivers[y].doa)
if intersection: if intersection:
print(intersection) logger.info(f"intersection: {intersection}")
receivers[x].d_2_last_intersection.append(v.haversine( receivers[x].d_2_last_intersection.append(v.haversine(
receivers[x].latitude, receivers[x].longitude, *intersection)) receivers[x].latitude, receivers[x].longitude,
*intersection))
receivers[y].d_2_last_intersection.append(v.haversine( receivers[y].d_2_last_intersection.append(v.haversine(
receivers[y].latitude, receivers[y].longitude, *intersection)) receivers[y].latitude, receivers[y].longitude,
*intersection))
intersection = list(intersection) intersection = list(intersection)
avg_conf = np.mean( avg_conf = np.mean(
[receivers[x].confidence, receivers[y].confidence]) [receivers[x].confidence, receivers[y].confidence])
...@@ -1080,7 +1177,8 @@ def run_receiver(receivers): ...@@ -1080,7 +1177,8 @@ def run_receiver(receivers):
intersect_list = np.concatenate( intersect_list = np.concatenate(
(intersect_list, intersection), axis=0) (intersect_list, intersection), axis=0)
# 11.01.2023::s1z: I don't know if this shit-code is blocking # 11.01.2023::s1z: I don't know if this shit-code is blocking
# or not. But if it is, you better fucking sleep # or not.
# But if it is, you better fucking sleep
# 10ms after each cycle to prevent 100% load # 10ms after each cycle to prevent 100% load
# of the CPU. # of the CPU.
time.sleep(0.001) time.sleep(0.001)
...@@ -1095,25 +1193,29 @@ def run_receiver(receivers): ...@@ -1095,25 +1193,29 @@ def run_receiver(receivers):
intersect_list[:, 0:3], weights=intersect_list[:, 2], axis=0) intersect_list[:, 0:3], weights=intersect_list[:, 2], axis=0)
keep, in_aoi = check_aoi(*avg_coord[0:2]) keep, in_aoi = check_aoi(*avg_coord[0:2])
if keep: if keep:
to_table = [receivers[x].doa_time, round(avg_coord[0], 6), round(avg_coord[1], 6), to_table = [receivers[x].doa_time, round(avg_coord[0], 6),
round(avg_coord[1], 6),
len(intersect_list), avg_coord[2], in_aoi] len(intersect_list), avg_coord[2], in_aoi]
command = '''INSERT INTO intersects command = """INSERT INTO intersects
(time, latitude, longitude, num_parents, confidence, aoi_id) (time, latitude, longitude, num_parents, confidence, aoi_id)
VALUES (?,?,?,?,?,?)''' VALUES (?,?,?,?,?,?)"""
DATABASE_EDIT_Q.put((command, (to_table,), True)) DATABASE_EDIT_Q.put((command, (to_table,), True))
DATABASE_RETURN.get(timeout=1) DATABASE_RETURN.get(timeout=1)
# Loop to compute intersections for a single receiver and update all receivers # Loop to compute intersections for a single receiver
# and update all receivers
for rx in receivers: for rx in receivers:
if (rx.isSingle and rx.isMobile and rx.isActive and if (rx.isSingle and rx.isMobile and rx.isActive and
rx.confidence >= ms.min_conf and rx.confidence >= ms.min_conf and
rx.power >= ms.min_power and rx.power >= ms.min_power and
rx.doa_time >= rx.previous_doa_time + 10000): rx.doa_time >= rx.previous_doa_time + 10000):
current_doa = [rx.doa_time, rx.station_id, rx.latitude, current_doa = [rx.doa_time, rx.station_id, rx.latitude,
rx.longitude, rx.confidence, rx.doa] rx.longitude, rx.confidence, rx.doa]
min_time = rx.doa_time - 1200000 # 15 Minutes min_time = rx.doa_time - 1200000 # 15 Minutes
c.execute('''SELECT latitude, longitude, confidence, lob FROM lobs c.execute("""SELECT latitude, longitude, confidence, lob
WHERE station_id = ? AND time > ?''', [rx.station_id, min_time]) FROM lobs
WHERE station_id = ? AND time > ?""",
[rx.station_id, min_time])
lob_array = c.fetchall() lob_array = c.fetchall()
current_time = current_doa[0] current_time = current_doa[0]
lat_rxa = current_doa[2] lat_rxa = current_doa[2]
...@@ -1133,7 +1235,8 @@ def run_receiver(receivers): ...@@ -1133,7 +1235,8 @@ def run_receiver(receivers):
if (spacial_diversity > min_diversity and if (spacial_diversity > min_diversity and
abs(doa_rxa - doa_rxb) > 5): abs(doa_rxa - doa_rxb) > 5):
intersection = plot_intersects(lat_rxa, lon_rxa, intersection = plot_intersects(lat_rxa, lon_rxa,
doa_rxa, lat_rxb, lon_rxb, doa_rxb) doa_rxa, lat_rxb,
lon_rxb, doa_rxb)
if intersection: if intersection:
intersection = list(intersection) intersection = list(intersection)
avg_conf = np.mean([conf_rxa, conf_rxb]) avg_conf = np.mean([conf_rxa, conf_rxb])
...@@ -1141,15 +1244,19 @@ def run_receiver(receivers): ...@@ -1141,15 +1244,19 @@ def run_receiver(receivers):
keep, in_aoi = check_aoi(*intersection[0:2]) keep, in_aoi = check_aoi(*intersection[0:2])
if keep: if keep:
keep_count += 1 keep_count += 1
to_table = [current_time, round(intersection[0], 5), round(intersection[1], 5), to_table = [current_time,
round(intersection[0], 5),
round(intersection[1], 5),
1, intersection[2], in_aoi] 1, intersection[2], in_aoi]
command = '''INSERT INTO intersects command = """INSERT INTO intersects
(time, latitude, longitude, num_parents, confidence, aoi_id) (time, latitude, longitude,
VALUES (?,?,?,?,?,?)''' num_parents, confidence,
DATABASE_EDIT_Q.put( aoi_id)
(command, (to_table,), True)) VALUES (?,?,?,?,?,?)"""
DATABASE_EDIT_Q.put((command, (to_table,),
True))
DATABASE_RETURN.get(timeout=1) DATABASE_RETURN.get(timeout=1)
print(f"Computed and kept {keep_count} intersections.") logger.info(f"Computed and kept {keep_count} intersections.")
command = "INSERT INTO lobs VALUES (?,?,?,?,?,?)" command = "INSERT INTO lobs VALUES (?,?,?,?,?,?)"
DATABASE_EDIT_Q.put((command, [current_doa, ], True)) DATABASE_EDIT_Q.put((command, [current_doa, ], True))
...@@ -1172,15 +1279,13 @@ def run_receiver(receivers): ...@@ -1172,15 +1279,13 @@ def run_receiver(receivers):
dots = 1 dots = 1
else: else:
dots += 1 dots += 1
clear(debugging)
conn.close() conn.close()
############################################### # noinspection SqlDialectInspection
# Checks if intersection should be kept or not
###############################################
def check_aoi(lat, lon): def check_aoi(lat, lon):
""" Checks if intersection should be kept or not """
keep_list = [] keep_list = []
in_aoi = None in_aoi = None
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
...@@ -1215,73 +1320,83 @@ def check_aoi(lat, lon): ...@@ -1215,73 +1320,83 @@ def check_aoi(lat, lon):
return keep, in_aoi return keep, in_aoi
############################################### def fetch_first_or_none(query):
# Adds a new receiver to the program, saves it """ Safe fetch one """
# in the database. result = query.fetchone()
############################################### if result is not None and len(result) > 0:
def add_receiver(receiver_url): return result[0]
return None
# noinspection SqlDialectInspection
def add_receiver(receiver_url, station_name=None):
""" Adds a new receiver to the program, saves it in the database. """
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
c = conn.cursor() c = conn.cursor()
try: try:
if any(x.station_url == receiver_url for x in receivers): if any(x.station_url == receiver_url for x in receivers):
print("Duplicate receiver, ignoring.") logger.warning("Duplicate receiver, ignoring.")
else: else:
receivers.append(receiver(receiver_url)) new_receiver = receiver(receiver_url, station_name)
new_rx = receivers[-1].receiver_dict() receivers.append(new_receiver)
to_table = [new_rx['station_id'], new_rx['station_url'], new_rx['auto'], new_rx = new_receiver.receiver_dict()
new_rx['mobile'], new_rx['single'], new_rx['latitude'], new_rx['longitude']] to_table = [new_rx['station_id'], new_rx['station_url'],
new_rx['auto'],
new_rx['mobile'], new_rx['single'], new_rx['latitude'],
new_rx['longitude']]
command = "INSERT OR IGNORE INTO receivers VALUES (?,?,?,?,?,?,?)" command = "INSERT OR IGNORE INTO receivers VALUES (?,?,?,?,?,?,?)"
DATABASE_EDIT_Q.put((command, [to_table, ], True)) DATABASE_EDIT_Q.put((command, [to_table, ], True))
DATABASE_RETURN.get(timeout=1) DATABASE_RETURN.get(timeout=1)
DATABASE_EDIT_Q.put(("done", None, False)) DATABASE_EDIT_Q.put(("done", None, False))
mobile = c.execute("SELECT isMobile FROM receivers WHERE station_id = ?", mobile = fetch_first_or_none(
[new_rx['station_id']]).fetchone()[0] c.execute("SELECT isMobile FROM receivers "
single = c.execute("SELECT isSingle FROM receivers WHERE station_id = ?", "WHERE station_id = ?", [new_rx['station_id']])
[new_rx['station_id']]).fetchone()[0] )
receivers[-1].isMobile = bool(mobile) single = fetch_first_or_none(
receivers[-1].isSingle = bool(single) c.execute("SELECT isSingle FROM receivers "
print("Created new DF Station at " + receiver_url) "WHERE station_id = ?", [new_rx['station_id']])
)
if mobile is not None:
new_receiver.isMobile = bool(mobile)
if single is not None:
new_receiver.isSingle = bool(single)
logger.info("Created new DF Station at " + receiver_url)
except AttributeError: except AttributeError:
pass pass
conn.close() conn.close()
############################################### # noinspection PyShadowingNames
# Reads receivers from the database into the
# program.
###############################################
def read_rx_table(): def read_rx_table():
""" Reads receivers from the database into the program. """
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
c = conn.cursor() c = conn.cursor()
try: try:
c.execute("SELECT station_url FROM receivers") c.execute("SELECT station_url, station_id FROM receivers")
rx_list = c.fetchall() rx_list = c.fetchall()
for x in rx_list: for data in rx_list:
receiver_url = x[0].replace('\n', '') station_url, station_id = [param.strip() for param in data]
add_receiver(receiver_url) add_receiver(station_url, station_id)
except sqlite3.OperationalError: except sqlite3.OperationalError:
rx_list = [] rx_list = []
conn.close() conn.close()
############################################### # noinspection SqlDialectInspection
# Updates the database with any changes made to
# the receivers.
###############################################
def update_rx_table(): def update_rx_table():
""" Updates the database with any changes made to the receivers. """
for item in receivers: for item in receivers:
rx = item.receiver_dict() rx = item.receiver_dict()
to_table = [rx['auto'], rx['mobile'], rx['single'], to_table = [rx['auto'], rx['mobile'], rx['single'],
rx['latitude'], rx['longitude'], rx['station_id']] rx['latitude'], rx['longitude'], rx['station_id']]
command = '''UPDATE receivers SET command = """UPDATE receivers SET isAuto=?,
isAuto=?, isMobile=?,
isMobile=?, isSingle=?,
isSingle=?, latitude=?,
latitude=?, longitude=?
longitude=? WHERE station_id = ?"""
WHERE station_id = ?'''
DATABASE_EDIT_Q.put((command, [to_table, ], True)) DATABASE_EDIT_Q.put((command, [to_table, ], True))
# try: # try:
DATABASE_RETURN.get(timeout=1) DATABASE_RETURN.get(timeout=1)
...@@ -1290,10 +1405,9 @@ def update_rx_table(): ...@@ -1290,10 +1405,9 @@ def update_rx_table():
DATABASE_EDIT_Q.put(("done", None, False)) DATABASE_EDIT_Q.put(("done", None, False))
############################################### # noinspection SqlDialectInspection
# Updates the database with new interest areas.
###############################################
def add_aoi(aoi_type, lat, lon, radius): def add_aoi(aoi_type, lat, lon, radius):
""" Updates the database with new interest areas. """
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
c = conn.cursor() c = conn.cursor()
...@@ -1307,10 +1421,8 @@ def add_aoi(aoi_type, lat, lon, radius): ...@@ -1307,10 +1421,8 @@ def add_aoi(aoi_type, lat, lon, radius):
DATABASE_EDIT_Q.put(("done", None, False)) DATABASE_EDIT_Q.put(("done", None, False))
#########################################
# Read all the AOIs from the DB
#########################################
def fetch_aoi_data(): def fetch_aoi_data():
""" Read all the AOIs from the DB """
conn = sqlite3.connect(database_name) conn = sqlite3.connect(database_name)
c = conn.cursor() c = conn.cursor()
c.execute('SELECT * FROM interest_areas') c.execute('SELECT * FROM interest_areas')
...@@ -1319,74 +1431,114 @@ def fetch_aoi_data(): ...@@ -1319,74 +1431,114 @@ def fetch_aoi_data():
return aoi_list return aoi_list
############################################### class DatabaseWriter(threading.Thread):
# One thread responsible for all database write """ Database Writer thread """
# operations.
############################################### def __init__(self, db_name):
def database_writer(): super().__init__(name=self.__class__.__name__, daemon=True)
conn = sqlite3.connect(database_name) self.database_name = db_name
c = conn.cursor() self._is_running = False
c.execute('''CREATE TABLE IF NOT EXISTS receivers ( self._is_running_lock = threading.Lock()
station_id TEXT UNIQUE,
station_url TEXT, @property
isAuto INTEGER, def is_running(self):
isMobile INTEGER, """ Get _is_running safely """
isSingle INTEGER, with self._is_running_lock:
latitude REAL, return self._is_running
longitude REAL)
''') @is_running.setter
c.execute('''CREATE TABLE IF NOT EXISTS interest_areas ( def is_running(self, value: bool):
uid INTEGER, """ set _is_running safely """
aoi_type TEXT, with self._is_running_lock:
latitude REAL, self._is_running = value
longitude REAL,
radius INTEGER) def stop(self):
''') """ Stop the Thread """
c.execute('''CREATE TABLE IF NOT EXISTS intersects ( self.is_running = False
id INTEGER PRIMARY KEY AUTOINCREMENT,
time INTEGER, def start(self):
latitude REAL, """ Start thread """
longitude REAL, self.is_running = True
num_parents INTEGER, super().start()
confidence INTEGER,
aoi_id INTEGER)''') # noinspection SqlDialectInspection
c.execute('''CREATE TABLE IF NOT EXISTS lobs (time INTEGER, def run(self):
station_id TEXT, """ One thread responsible for all database write operations. """
latitude REAL, conn = sqlite3.connect(self.database_name)
longitude REAL, c = conn.cursor()
confidence INTEGER, c.execute("""CREATE TABLE IF NOT EXISTS receivers (
lob REAL)''') station_id TEXT UNIQUE,
conn.commit() station_url TEXT,
while True: isAuto INTEGER,
# items should be list of lists isMobile INTEGER,
command, items, reply = DATABASE_EDIT_Q.get() isSingle INTEGER,
if command == "done": latitude REAL,
conn.commit() longitude REAL)
elif command == "close": """)
conn.commit() c.execute("""CREATE TABLE IF NOT EXISTS interest_areas (
conn.close() uid INTEGER,
if reply: aoi_type TEXT,
DATABASE_RETURN.put(True) latitude REAL,
break longitude REAL,
else: radius INTEGER)
c.executemany(command, items) """)
if reply: c.execute("""CREATE TABLE IF NOT EXISTS intersects (
DATABASE_RETURN.put(True) id INTEGER PRIMARY KEY AUTOINCREMENT,
time INTEGER,
latitude REAL,
longitude REAL,
num_parents INTEGER,
confidence INTEGER,
aoi_id INTEGER)""")
c.execute("""CREATE TABLE IF NOT EXISTS lobs (time INTEGER,
station_id TEXT,
latitude REAL,
longitude REAL,
confidence INTEGER,
lob REAL)""")
conn.commit()
while self.is_running:
# items should be list of lists
command, items, reply = DATABASE_EDIT_Q.get()
if command == "done":
conn.commit()
elif command == "close":
conn.commit()
conn.close()
if reply:
DATABASE_RETURN.put(True)
break
else:
c.executemany(command, items)
if reply:
DATABASE_RETURN.put(True)
time.sleep(0.001)
###############################################
# Thangs to do before closing the program.
###############################################
def finish(): def finish():
clear(debugging) """ Thangs to do before closing the program. """
print("Processing, please wait.") logger.info("Processing df-aggregator clean-up tasks")
ms.receiving = False ms.receiving = False
update_rx_table() update_rx_table()
DATABASE_EDIT_Q.put(("close", None, True)) DATABASE_EDIT_Q.put(("close", None, True))
DATABASE_RETURN.get(timeout=1) DATABASE_RETURN.get(timeout=1)
if geofile is not None: if geofile is not None:
write_geojson(*process_data(database_name)[:2]) write_geojson(*process_data(database_name)[:2])
kill(getpid(), signal.SIGTERM) # kill(getpid(), signal.SIGTERM)
logger.info("clean-up has done")
def stop_server(server):
""" Stop server and execute finish gracefuly """
def wr(_signum, _frame):
""" We need a wraper to dynamicaly set server """
server.stop()
server.join()
finish()
return wr
if __name__ == '__main__': if __name__ == '__main__':
...@@ -1401,32 +1553,54 @@ if __name__ == '__main__': ...@@ -1401,32 +1553,54 @@ if __name__ == '__main__':
help="List of receiver URLs", metavar="FILE") help="List of receiver URLs", metavar="FILE")
parser.add_option("-g", "--geofile", dest="geofile", parser.add_option("-g", "--geofile", dest="geofile",
help="GeoJSON Output File", metavar="FILE") help="GeoJSON Output File", metavar="FILE")
parser.add_option("-e", "--epsilon", dest="eps", help="Max Clustering Distance, Default \"auto\".", parser.add_option("-e", "--epsilon", dest="eps",
help="Max Clustering Distance, Default \"auto\".",
metavar="NUMBER or \"auto\"", default="auto") metavar="NUMBER or \"auto\"", default="auto")
parser.add_option("-c", "--confidence", dest="conf", help="Minimum confidence value, default 10", parser.add_option("-c", "--confidence", dest="conf",
help="Minimum confidence value, default 10",
metavar="NUMBER", type="int", default=10) metavar="NUMBER", type="int", default=10)
parser.add_option("-p", "--power", dest="pwr", help="Minimum power value, default 10", parser.add_option("-p", "--power", dest="pwr",
help="Minimum power value, default 10",
metavar="NUMBER", type="int", default=10) metavar="NUMBER", type="int", default=10)
parser.add_option("-m", "--min-samples", dest="minsamp", help="Minimum samples per cluster. Default: \"auto\"", parser.add_option("-m", "--min-samples", dest="minsamp",
help="Minimum samples per cluster. Default: \"auto\"",
metavar="NUMBER or \"auto\"", default="auto") metavar="NUMBER or \"auto\"", default="auto")
parser.add_option("--plot_intersects", dest="plotintersects", help="""Plots all the intersect points in a cluster. parser.add_option("--plot_intersects", dest="plotintersects",
Only applies when clustering is turned on. This creates larger CZML files.""", action="store_true") help=("Plots all the intersect points in a cluster. "
parser.add_option("-o", "--offline", dest="disable", help="Starts program with receiver turned off.", "Only applies when clustering is turned on. "
"This creates larger CZML files."),
action="store_true")
parser.add_option("-o", "--offline", dest="disable",
help="Starts program with receiver turned off.",
action="store_false", default=True) action="store_false", default=True)
parser.add_option("--access_token", dest="token_file", parser.add_option("--access_token", dest="token_file",
help="Cesium Access Token File", metavar="FILE") help="Cesium Access Token File", metavar="FILE")
parser.add_option("--ip", dest="ipaddr", help="IP Address to serve from. Default 127.0.0.1", parser.add_option("--ip", dest="ipaddr",
help="IP Address to serve from. Default 127.0.0.1",
metavar="IP ADDRESS", type="str", default="127.0.0.1") metavar="IP ADDRESS", type="str", default="127.0.0.1")
parser.add_option("--port", dest="port", help="Port number to serve from. Default 8080", parser.add_option("--port", dest="port",
help="Port number to serve from. Default 8080",
metavar="NUMBER", type="int", default=8080) metavar="NUMBER", type="int", default=8080)
parser.add_option("--debug", dest="debugging", help="Does not clear the screen. Useful for seeing errors and warnings.", parser.add_option("--debug", dest="debugging",
help="Does not clear the screen. "
"Useful for seeing errors and warnings.",
action="store_true") action="store_true")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
# INIT LOGGING
logging_level = logging.DEBUG
if not options.debugging: # Limit some spam
# noinspection SpellCheckingInspection
logging.getLogger("geventwebsocket.handler").level = logging.WARNING
logging.getLogger("asyncio").level = logging.INFO
logging_level = logging.INFO
init_logging(level=logging_level)
#
mandatories = ['database_name'] mandatories = ['database_name']
for m in mandatories: for m in mandatories:
if options.__dict__[m] is None: if options.__dict__[m] is None:
print("You forgot an arguement") logger.error("You forgot an arguement")
parser.print_help() parser.print_help()
exit(-1) exit(-1)
...@@ -1447,41 +1621,38 @@ if __name__ == '__main__': ...@@ -1447,41 +1621,38 @@ if __name__ == '__main__':
else: else:
access_token = None access_token = None
web = threading.Thread(target=start_server, web = BottleWebServer(options.ipaddr, options.port)
args=(options.ipaddr, options.port))
web.daemon = True
web.start() web.start()
dbwriter = threading.Thread(target=database_writer) db_writer = DatabaseWriter(database_name)
dbwriter.daemon = True db_writer.start()
dbwriter.start()
try: ###############################################
############################################### # Reds receivers from the database first, then
# Reds receivers from the database first, then # loads receivers in from a user provided list.
# loads receivers in from a user provided list. ###############################################
############################################### read_rx_table()
read_rx_table() if rx_file:
if rx_file: with open(rx_file, "r") as file2:
with open(rx_file, "r") as file2: for receiver_url in file2.readlines():
receiver_list = file2.readlines() add_receiver(receiver_url.strip())
for x in receiver_list:
receiver_url = x.replace('\n', '') ###############################################
add_receiver(receiver_url) # Run the main loop!
###############################################
############################################### # while True:
# Run the main loop! # if ms.receiving:
############################################### # run_receiver(receivers)
while True: # if not debugging:
if ms.receiving: # print("Receiver Paused")
run_receiver(receivers) # # 11.01.2023::s1z: I don't know wtf is this crap,
clear(debugging) # # but sleep 1 sec,
if not debugging: # # You gotta be kidding me.
print("Receiver Paused") # # Changed from 1 sec to 10 ms.
# 11.01.2023::s1z: I don't know wtf is this crap, but sleep 1 sec, # time.sleep(0.001)
# You gotta be kidding me.
# Changed fro 1 sec to 10 ms. ether_service = EtherService(web, db_writer, receivers, shared_dict)
time.sleep(0.01) signal.signal(signal.SIGINT, stop_server(ether_service))
signal.signal(signal.SIGTERM, stop_server(ether_service))
except KeyboardInterrupt: ether_service.start()
finish() ether_service.join()
requirements.txt
...@@ -5,3 +5,8 @@ numpy>=1.13.3 ...@@ -5,3 +5,8 @@ numpy>=1.13.3
lxml>=4.2.1 lxml>=4.2.1
czml3>=0.5.4 czml3>=0.5.4
scikit_learn>=0.23.2 scikit_learn>=0.23.2
# ether requirements
absl-py==1.4.0
aiohttp==3.8.4
simplejson==3.18.4
utils/ether_service/entities/receiver.py 0 → 100644
""" This is a Protocol to wirk with obsolete DF-Aggregator objects """
from typing import Protocol
class Receiver(Protocol):
""" Receiver object """
...
def update(self, first_run=False):
""" Some update logic """
...
def receiver_dict(self):
""" Receiver representation in json format """
...
def lob_length(self):
""" Some magic here, i suppose. We skip it for now.
There's a bit of hardcode with 200... """
...
utils/ether_service/ether_service.py 0 → 100644
""" Ether web service """
import asyncio
import logging
from threading import Thread, Lock
from typing import List, Dict
from aiohttp import web
from utils.ether_service.entities.receiver import Receiver
from utils.ether_service.handlers.api.v1.ws import WSHandler
from utils.run_app_fixed import create_task
logger = logging.getLogger(__name__)
class EtherService(Thread):
""" Ether Service. This is a thread that runs aiohttp.
Main purpose is:
1. Receive data from KrSPIs.
2. Do math on the background.
3. Update global variables for the obsolete web service."""
def __init__(self, web_service: Thread, db_writer: Thread,
receivers: List[Receiver], shared_dict: Dict,
*,
interface="0.0.0.0", port=8090):
super().__init__(name=self.__class__.__name__, daemon=True)
self.web_service = web_service
self.db_writer = db_writer
self.receivers = receivers
self.shared_dict = shared_dict
self._is_running = False
self._is_running_lock = Lock()
self.interface = interface
self.port = port
self.io_loop = asyncio.get_event_loop()
self.web_app = web.Application()
self.web_app.add_routes([
WSHandler().compile(),
])
self.main_task = None
@property
def is_running(self):
""" Get _is_running safely """
with self._is_running_lock:
return self._is_running
@is_running.setter
def is_running(self, value: bool):
""" set _is_running safely """
with self._is_running_lock:
self._is_running = value
def run(self):
""" Thread routine """
self.is_running = True
self.main_task = create_task(self.web_app, host=self.interface,
port=self.port, loop=self.io_loop,
handle_signals=False,
print=lambda x: x)
logger.info("Server has started")
self.io_loop.run_forever()
logger.info("Server has stopped")
def start(self):
""" Start the app """
logger.info("Server is starting")
super().start()
def stop(self):
""" Stop the app """
logger.info("Server is stopping")
self.is_running = False
# add web_server.stop()
# add db_writer.stop()
self.io_loop.call_soon_threadsafe(self.main_task.cancel)
self.io_loop.call_soon_threadsafe(self.io_loop.stop)
utils/ether_service/handlers/api/v1/ws.py 0 → 100644
""" Websocket handler """
import base64
import sys
from typing import Any, Awaitable, Union, Tuple, Optional
import aiohttp
import uuid
from absl import logging
from aiohttp import WSMessage
from aiohttp.web_exceptions import HTTPForbidden
from aiohttp.web_request import Request
from aiohttp.web_response import Response
from aiohttp.web_ws import WebSocketResponse
from multidict import CIMultiDictProxy
from ...base_handler import BaseHandler
class AuthType:
""" Supported auth types """
BASIC = "basic"
BEARER = "bearer"
async def auth_basic(credentials: str) -> bool:
""" Auth using basic credentials """
# noinspection PyBroadException
try:
data = base64.b64decode(credentials.encode("utf-8")).decode("utf-8")
login, password = data.split(":")
if login == "kraken" and password == "kraken":
return True
except Exception:
logging.error("auth_basic error:", exc_info=sys.exc_info())
return False
def parse_auth_header(headers: "CIMultiDictProxy[str]")\
-> Tuple[Optional[str], Optional[str]]:
""" Parse auth header, return AuthType and AuthData """
auth_header = headers.get("Authorization") or None
if auth_header is not None:
try:
auth_type, auth_data = auth_header.split(" ")
return auth_type, auth_data
except ValueError:
logging.error(f"Could not parse Auth Header: '{auth_header}'")
return None, None
async def authenticate(request: Request) -> bool:
""" Authenticate request """
auth_type, auth_data = parse_auth_header(request.headers)
if auth_type is not None and auth_data is not None:
logging.info(f"auth_type and auth_data are fine, handling")
if auth_type.lower() == AuthType.BASIC:
logging.info(f"Handling Basic auth")
return await auth_basic(auth_data)
elif auth_data.lower() == AuthType.BEARER:
logging.warning(f"{AuthType.BEARER} is not implemented!")
return False
else:
logging.warning(f"{auth_type} is not supported!")
logging.warning(f"Either, auth_type or auth_data is null")
return False
def auth_required(f) -> Any:
""" Check if user is auth """
async def wr(self, request: Request, *args, **kwargs)\
-> Awaitable[Union[WebSocketResponse, Response]]:
""" wrapper """
is_auth = await authenticate(request) or self.PATH.find("?") >= 0
if is_auth:
return await f(self, request, *args, **kwargs)
raise HTTPForbidden()
return wr
class WSHandler(BaseHandler):
""" DoA Handler """
METHOD = "GET"
PATH = "/api/v1/ws"
ws = None
connection_id = None
@staticmethod
async def handle_binary(data: bytes) -> None:
""" handle web socket data """
logging.info(f"Binary data received: {data}")
async def handle_message(self, msg: WSMessage) -> None:
""" Handle incoming websocket packet """
if msg.type == aiohttp.WSMsgType.ERROR:
ex = self.ws.exception()
exc_info = (ex.__class__, ex, ex.__traceback__)
logging.error("Ws connection closed", exc_info=exc_info)
elif msg.type == aiohttp.WSMsgType.BINARY:
await self.handle_binary(msg.data)
elif msg.type == aiohttp.WSMsgType.TEXT:
await self.ws.send_str(msg.data)
@auth_required
async def handler(self, request: Request) -> WebSocketResponse:
""" The handler """
connection_id = uuid.uuid4()
logging.info(f"WS opened. connection_id: '{connection_id}'")
self.ws = ws = WebSocketResponse()
await ws.prepare(request)
logging.info(f"WS prepared. connection_id: '{connection_id}'")
async for msg in ws:
# noinspection PyTypeChecker
await self.handle_message(msg)
logging.info(f"WS closed. connection_id: '{connection_id}'")
return ws
utils/ether_service/handlers/base_handler.py 0 → 100644
""" This is a base handler """
import logging
from aiohttp import web
from aiohttp.web_request import Request
from aiohttp.web_response import Response
class BaseHandler:
""" API V1 Health Check """
METHOD = None
PATH = None
async def handler(self, _: Request) -> Response:
""" The handler """
raise NotImplementedError()
def compile(self):
""" compile into the route """
logger = logging.getLogger(self.__module__)
logger.info(f"{self.METHOD}:{self.PATH}")
return web.route(self.METHOD, self.PATH, self.handler)
def _get_name(self):
""" Get handler name """
return f"{self.__module__}.{self.__class__.__name__}"
utils/log.py 0 → 100644
from __future__ import absolute_import
from __future__ import unicode_literals
import logging
def init_logging(filename=None, level=None):
""" init logging on the app level """
logging_config = {"format": "%(asctime)-23s %(levelname)8s::"
"%(name)s::%(funcName)s:"
"%(message)s",
"level": level or logging.DEBUG}
if filename is not None:
logging_config["filename"] = filename
logging.getLogger().handlers = []
logging.basicConfig(**logging_config)
utils/run_app_fixed.py 0 → 100644
""" Default run_app uses method run_until_complete,
which is awful and wrongn """
import asyncio
import logging
import socket
from asyncio import all_tasks, Task
from ssl import SSLContext
from typing import Union, Awaitable, Optional
from aiohttp.abc import Application, AbstractAccessLogger
from aiohttp.log import access_logger
from aiohttp.web import HostSequence, _run_app, _cancel_tasks
from typing import (
Any,
Awaitable,
Callable,
Iterable as TypingIterable,
List,
Optional,
Set,
Type,
Union,
cast,
)
from aiohttp.web_log import AccessLogger
from aiohttp.web_runner import GracefulExit
def create_task(
app: Union[Application, Awaitable[Application]],
*,
host: Optional[Union[str, HostSequence]] = None,
port: Optional[int] = None,
path: Optional[str] = None,
sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None,
shutdown_timeout: float = 60.0,
keepalive_timeout: float = 75.0,
ssl_context: Optional[SSLContext] = None,
print: Callable[..., None] = print,
backlog: int = 128,
access_log_class: Type[AbstractAccessLogger] = AccessLogger,
access_log_format: str = AccessLogger.LOG_FORMAT,
access_log: Optional[logging.Logger] = access_logger,
handle_signals: bool = True,
reuse_address: Optional[bool] = None,
reuse_port: Optional[bool] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
) -> Task:
"""Run an app locally"""
if loop is None:
loop = asyncio.new_event_loop()
# Configure if and only if in debugging mode and using the default logger
if loop.get_debug() and access_log and access_log.name == "aiohttp.access":
if access_log.level == logging.NOTSET:
access_log.setLevel(logging.DEBUG)
if not access_log.hasHandlers():
access_log.addHandler(logging.StreamHandler())
main_task = loop.create_task(
_run_app(
app,
host=host,
port=port,
path=path,
sock=sock,
shutdown_timeout=shutdown_timeout,
keepalive_timeout=keepalive_timeout,
ssl_context=ssl_context,
print=print,
backlog=backlog,
access_log_class=access_log_class,
access_log_format=access_log_format,
access_log=access_log,
handle_signals=handle_signals,
reuse_address=reuse_address,
reuse_port=reuse_port,
)
)
return main_task
def run_app(
app: Union[Application, Awaitable[Application]],
*,
host: Optional[Union[str, HostSequence]] = None,
port: Optional[int] = None,
path: Optional[str] = None,
sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None,
shutdown_timeout: float = 60.0,
keepalive_timeout: float = 75.0,
ssl_context: Optional[SSLContext] = None,
print: Callable[..., None] = print,
backlog: int = 128,
access_log_class: Type[AbstractAccessLogger] = AccessLogger,
access_log_format: str = AccessLogger.LOG_FORMAT,
access_log: Optional[logging.Logger] = access_logger,
handle_signals: bool = True,
reuse_address: Optional[bool] = None,
reuse_port: Optional[bool] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
) -> None:
"""Run an app locally"""
main_task = create_task(app, host=host, port=port, path=path, sock=sock,
shutdown_timeout=shutdown_timeout,
keepalive_timeout=keepalive_timeout,
ssl_context=ssl_context, print=print,
backlog=backlog,
access_log_class=access_log_class,
access_log_format=access_log_format,
access_log=access_log,
handle_signals=handle_signals,
reuse_address=reuse_address,
reuse_port=reuse_port, loop=loop)
try:
asyncio.set_event_loop(loop)
loop.run_forever()
except (GracefulExit, KeyboardInterrupt): # pragma: no cover
pass
finally:
_cancel_tasks({main_task}, loop)
_cancel_tasks(all_tasks(loop), loop)
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment