Initial Commit

.gitignore

+__pycache__/*
+*.kml
+CesiumJS/
+*.bak
+*.geojson
+test*
+*.db
+*.txt

df-aggregator.py

+#!/usr/bin/env python3
+
+import vincenty as v
+import numpy as np
+import math
+import time
+import sqlite3
+from optparse import OptionParser
+from os import system, name
+from lxml import etree
+from sklearn.cluster import DBSCAN
+from sklearn.preprocessing import StandardScaler
+from geojson import Point, MultiPoint, Feature, FeatureCollection
+
+d = 40000 #meters
+
+all_pt_style = {"name": "Various Points", "marker-color": "#FF0000"}
+best_pt_style = {"name": "Most Likely TX Location", "marker-color": "#00FF00"}
+
+class receiver:
+    def __init__(self, station_url):
+        self.station_url = station_url
+        try:
+            self.update()
+        except:
+            print("Problem connecting to receiver.")
+            quit()
+
+    def update(self):
+        try:
+            xml_contents = etree.parse(self.station_url)
+            xml_station_id = xml_contents.find('STATION_ID')
+            self.station_id = xml_station_id.text
+            xml_doa_time = xml_contents.find('TIME')
+            self.doa_time = int(xml_doa_time.text)
+            xml_freq = xml_contents.find('FREQUENCY')
+            self.frequency = float(xml_freq.text)
+            xml_latitude = xml_contents.find('LOCATION/LATITUDE')
+            self.latitude = float(xml_latitude.text)
+            xml_longitude = xml_contents.find('LOCATION/LONGITUDE')
+            self.longitude = float(xml_longitude.text)
+            xml_heading = xml_contents.find('LOCATION/HEADING')
+            self.heading = float(xml_heading.text)
+            xml_doa = xml_contents.find('DOA')
+            self.raw_doa = float(xml_doa.text)
+            self.doa = self.heading + (360 - self.raw_doa)
+            if self.doa < 0:
+                self.doa += 360
+            elif self.doa > 359:
+                self.doa -= 360
+            xml_power = xml_contents.find('PWR')
+            self.power = float(xml_power.text)
+            xml_conf = xml_contents.find('CONF')
+            self.confidence = int(xml_conf.text)
+        except:
+            print("Problem connecting to receiver.")
+            quit()
+
+    latitude = 0.0
+    longitude = 0.0
+    heading = 0.0
+    raw_doa = 0.0
+    doa = 0.0
+    frequency = 0.0
+    power = 0.0
+    confidence = 0
+    doa_time = 0
+
+
+def plot_polar(lat_a, lon_a, lat_a2, lon_a2):
+    # Convert points in great circle 1, degrees to radians
+    p1_lat1_rad = math.radians(lat_a)
+    p1_long1_rad =  math.radians(lon_a)
+    p1_lat2_rad =  math.radians(lat_a2)
+    p1_long2_rad =  math.radians(lon_a2)
+
+    # Put in polar coordinates
+    x1 = math.cos(p1_lat1_rad) * math.cos(p1_long1_rad)
+    y1 = math.cos(p1_lat1_rad) * math.sin(p1_long1_rad)
+    z1 = math.sin(p1_lat1_rad)
+    x2 = math.cos(p1_lat2_rad) * math.cos(p1_long2_rad)
+    y2 = math.cos(p1_lat2_rad) * math.sin(p1_long2_rad)
+    z2 = math.sin(p1_lat2_rad)
+
+    return ([x1, y1, z1], [x2, y2, z2])
+
+# Find line of intersection between two planes
+# L = np.cross(N1, N2)
+def plot_intersects(lat_a, lon_a, doa_a, lat_b, lon_b, doa_b, max_distance = 50000):
+    # plot another point on the lob
+    # v.direct(lat_a, lon_a, doa_a, d)
+    # returns (lat_a2, lon_a2)
+
+    # Get normal to planes containing great circles
+    # np.cross product of vector to each point from the origin
+    coord_a2 = v.direct(lat_a, lon_a, doa_a, d)
+    coord_b2 = v.direct(lat_b, lon_b, doa_b, d)
+    plane_a = plot_polar(lat_a, lon_a, *coord_a2)
+    plane_b = plot_polar(lat_b, lon_b, *coord_b2)
+    N1 = np.cross(plane_a[0], plane_a[1])
+    N2 = np.cross(plane_b[0], plane_b[1])
+
+    # Find line of intersection between two planes
+    L = np.cross(N1, N2)
+    # Find two intersection points
+    X1 = L / np.sqrt(L[0]**2 + L[1]**2 + L[2]**2)
+    X2 = -X1
+    mag = lambda q: np.sqrt(np.vdot(q, q))
+    dist1 = mag(X1-plane_a[0])
+    dist2 = mag(X2-plane_a[0])
+    #return the (lon_lat pair of the closer intersection)
+    if dist1 < dist2:
+        i_lat = math.asin(X1[2]) * 180./np.pi
+        i_long = math.atan2(X1[1], X1[0]) * 180./np.pi
+    else:
+        i_lat = math.asin(X2[2]) * 180./np.pi
+        i_long = math.atan2(X2[1], X2[0]) * 180./np.pi
+
+    check_bearing = v.get_heading((lat_a, lon_a), (i_lat, i_long))
+
+    if abs(check_bearing - doa_a) < 5:
+        km = v.inverse([lat_a, lon_a], [i_lat, i_long])
+        if km[0] < max_distance:
+            return (i_lat, i_long)
+        else:
+            return None
+
+def process_data(database_name, outfile, eps, min_samp):
+    conn = sqlite3.connect(database_name)
+    c = conn.cursor()
+
+    intersect_list = []
+
+    c.execute("SELECT COUNT(*) FROM intersects")
+    n_intersects = int(c.fetchone()[0])
+    #print(n_intersects)
+    c.execute("SELECT latitude, longitude FROM intersects")
+    intersect_array = np.array(c.fetchall())
+    # print(intersect_array)
+    likely_location = []
+    best_point = []
+    if intersect_array.size != 0:
+        if eps > 0:
+            X = StandardScaler().fit_transform(intersect_array)
+
+            # Compute DBSCAN
+            db = DBSCAN(eps=eps, min_samples=min_samp).fit(X)
+            core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
+            core_samples_mask[db.core_sample_indices_] = True
+            labels = db.labels_
+
+            intersect_array = np.column_stack((intersect_array, labels))
+            # print(intersect_array)
+
+            # Number of clusters in labels, ignoring noise if present.
+            n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
+            n_noise_ = list(labels).count(-1)
+            clear()
+            print('Number of clusters: %d' % n_clusters_)
+            print('Outliers Removed: %d' % n_noise_)
+            # print(intersect_array)
+
+            for x in range(n_clusters_):
+                cluster = np.array([]).reshape(0,2)
+                for y in range(len(intersect_array)):
+                    if intersect_array[y][2] == x:
+                        cluster = np.concatenate((cluster, [intersect_array[y][0:2]]), axis = 0)
+                likely_location.append(Reverse(np.mean(cluster, axis=0).tolist()))
+                best_point = Feature(properties = best_pt_style, geometry = MultiPoint(tuple(likely_location)))
+                # for x in range(len(likely_location)):
+                #     print(Reverse(likely_location[x]))
+            for x in likely_location:
+                print(Reverse(x))
+
+        for i in range(len(intersect_array)):
+            try:
+                if intersect_array[i][2] >= 0:
+                    intersect_list.append(Reverse(intersect_array[i][0:2].tolist()))
+            except IndexError:
+                intersect_list.append(Reverse(intersect_array[i].tolist()))
+        #print(intersect_list)
+        all_the_points = Feature(properties = all_pt_style, geometry = MultiPoint(tuple(intersect_list)))
+
+        with open(outfile, "w") as file1:
+            if eps > 0:
+                file1.write(str(FeatureCollection([best_point, all_the_points])))
+            else:
+                file1.write(str(FeatureCollection([all_the_points])))
+        print(f"Wrote file {geofile}")
+
+    else:
+        print("No Intersections.")
+
+def Reverse(lst):
+    lst.reverse()
+    return lst
+
+def clear():
+      # for windows
+    if name == 'nt':
+        _ = system('cls')
+    # for mac and linux(here, os.name is 'posix')
+    else:
+        _ = system('clear')
+
+if __name__ == '__main__':
+    usage = "usage: %prog [options]"
+    parser = OptionParser(usage=usage)
+    parser.add_option("-g", "--geofile", dest="geofile", help="GeoJSON Output File", metavar="FILE")
+    parser.add_option("-r", "--receivers", dest="rx_file", help="List of receiver URLs", metavar="FILE")
+    parser.add_option("-d", "--database", dest="database_name", help="Database File", metavar="FILE")
+    parser.add_option("-e", "--epsilon", dest="eps", help="Max Clustering Distance, Default 0.2. 0 to disable clustering.",
+    metavar="FILE", type="float", default=0.2)
+    parser.add_option("-c", "--confidence", dest="conf", help="Minimun confidence value, default 10",
+    metavar="FILE", type="int", default=10)
+    parser.add_option("-p", "--power", dest="pwr", help="Minimun power value, default 10",
+    metavar="FILE", type="int", default=10)
+    parser.add_option("-m", "--min-samples", dest="minsamp", help="Minimun samples per cluster. Default 20",
+    metavar="FILE", type="int", default=20)
+    parser.add_option("--dist-from-reference", dest="mdfr", help="Max distance in km from intersection with strongest signal.",
+    metavar="FILE", type="int", default=500)
+    (options, args) = parser.parse_args()
+
+    mandatories = ['geofile', 'rx_file', 'database_name']
+    for m in mandatories:
+      if options.__dict__[m] is None:
+        print("You forgot an arguement")
+        parser.print_help()
+        exit(-1)
+
+    geofile = options.geofile
+    rx_file = options.rx_file
+    database_name = options.database_name
+    eps = options.eps
+    min_conf = options.conf
+    min_power = options.pwr
+    max_dist_from_ref = options.mdfr
+    min_samp = options.minsamp
+
+    try:
+        clear()
+        dots = 0
+        conn = sqlite3.connect(database_name)
+        c = conn.cursor()
+        c.execute("CREATE TABLE IF NOT EXISTS intersects (time INTEGER, latitude REAL, longitude REAL)")
+
+        receivers = []
+        with open(rx_file, "r") as file2:
+            receiver_list = file2.readlines()
+            for x in receiver_list:
+                #print(x.replace('\n', ''))
+                receivers.append(receiver(x.replace('\n', '')))
+
+        avg_list = []
+        average_intersects = np.array([]).reshape(0,2)
+
+        while True:
+            intersect_list = np.array([]).reshape(0,3)
+            for x in range(len(receivers)):
+                for y in range(x):
+                    if x != y:
+                        try:
+                            if receivers[x].confidence > min_conf and receivers[y].confidence > min_conf:
+                                intersection = list(plot_intersects(receivers[x].latitude, receivers[x].longitude,
+                                receivers[x].doa, receivers[y].latitude, receivers[y].longitude, receivers[y].doa))
+                                print(intersection)
+                                avg_pwr = np.mean([receivers[x].power, receivers[y].power])
+                                intersection.append(avg_pwr)
+                                intersection = np.array([intersection])
+                                # print(f"Intersect: {intersection}")
+                                if intersection.any() != None:
+                                    intersect_list = np.concatenate((intersect_list, intersection), axis=0)
+                                    #print(intersect_list)
+                        except TypeError: # I can't figure out what's causing me to need this here
+                            pass
+
+            pwr_list = []
+            delete_these = []
+            if intersect_list.size != 0:
+                #print(intersect_list)
+                if intersect_list.size > 1:
+                    for x in range(len(intersect_list)):
+                        pwr_list.append(intersect_list[x, 2])
+                    reference_pt = pwr_list.index(max(pwr_list))
+                    for x in range(len(intersect_list)):
+                        if x != reference_pt:
+                            # print(f"Checking point: {intersect_list[x]} against reference {intersect_list[reference_pt]}")
+                            dist = v.inverse(intersect_list[reference_pt, 0:2], intersect_list[x, 0:2])[0]
+                            if dist > max_dist_from_ref:
+                                delete_these.append(x)
+                    if not delete_these:
+                        for x in delete_these:
+                            intersect_list = np.delete(intersect_list, x, 0)
+                            # print("Deleted Bad Intersection")
+
+                avg_coord = np.mean(intersect_list, axis = 0)
+                to_table = [receivers[x].doa_time, avg_coord[0], avg_coord[1]]
+                # print(to_table)
+                c.execute("INSERT INTO intersects VALUES (?,?,?)", to_table)
+                conn.commit()
+
+            for rx in receivers:
+                rx.update()
+            time.sleep(1)
+            if dots > 5:
+                dots = 1
+            else:
+                dots += 1
+            clear()
+            print("Receiving" + dots*'.')
+            print("Press control+c to process data and exit.")
+
+    except KeyboardInterrupt:
+        clear()
+        print("Processing, please wait.")
+        conn.commit()
+        conn.close()
+        process_data(database_name, geofile, eps, min_samp)
+
+        quit()

vincenty.py

+#!/usr/bin/env python3
+# WGS 84
+#a = 6378137  # meters
+#f = 1 / 298.257223563
+#b = 6356752.314245  # meters; b = (1 - f)a
+#f: flattening of the ellipsoid
+#a: radius of the ellipsoid, meteres
+#phi1: latitude of the start point, decimal degrees
+#lembda1: longitude of the start point, decimal degrees
+#alpha12: bearing, decimal degrees
+#s: Distance to endpoint, meters
+import sys
+from math import atan
+from math import atan2
+from math import cos
+from math import radians
+from math import degrees
+from math import sin
+from math import sqrt
+from math import tan
+from math import pow
+
+a=6378137.0                             # radius at equator in meters (WGS-84)
+f=1/298.257223563                       # flattening of the ellipsoid (WGS-84)
+b=(1-f)*a
+
+def get_heading(coord1, coord2):
+     lat1 = radians(coord1[0])
+     lon1 = radians(coord1[1])
+     lat2 = radians(coord2[0])
+     lon2 = radians(coord2[1])
+     bearing_plot_X =  cos(lat2) * sin(lon2 - lon1)
+     bearing_plot_Y = cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(lon1 - lon2)
+     heading = degrees(atan2(bearing_plot_X,bearing_plot_Y))
+     if heading <0: heading += 360
+     return heading
+
+def inverse(coord1,coord2,maxIter=200,tol=10**-12):
+    global a
+    global f
+    global b
+
+    phi_1,L_1,=coord1
+    phi_2,L_2,=coord2
+
+    u_1=atan((1-f)*tan(radians(phi_1)))
+    u_2=atan((1-f)*tan(radians(phi_2)))
+
+    L=radians(L_2-L_1)
+
+    Lambda=L                                # set initial value of lambda to L
+
+    sin_u1=sin(u_1)
+    cos_u1=cos(u_1)
+    sin_u2=sin(u_2)
+    cos_u2=cos(u_2)
+
+    try:
+        iters=0
+        for i in range(0,maxIter):
+            iters+=1
+
+            cos_lambda=cos(Lambda)
+            sin_lambda=sin(Lambda)
+            sin_sigma=sqrt((cos_u2*sin(Lambda))**2+(cos_u1*sin_u2-sin_u1*cos_u2*cos_lambda)**2)
+            cos_sigma=sin_u1*sin_u2+cos_u1*cos_u2*cos_lambda
+            sigma=atan2(sin_sigma,cos_sigma)
+            sin_alpha=(cos_u1*cos_u2*sin_lambda)/sin_sigma
+            cos_sq_alpha=1-sin_alpha**2
+            cos2_sigma_m=cos_sigma-((2*sin_u1*sin_u2)/cos_sq_alpha)
+            C=(f/16)*cos_sq_alpha*(4+f*(4-3*cos_sq_alpha))
+            Lambda_prev=Lambda
+            Lambda=L+(1-C)*f*sin_alpha*(sigma+C*sin_sigma*(cos2_sigma_m+C*cos_sigma*(-1+2*cos2_sigma_m**2)))
+
+            # successful convergence
+            diff=abs(Lambda_prev-Lambda)
+            if diff<=tol:
+                break
+
+        u_sq=cos_sq_alpha*((a**2-b**2)/b**2)
+        A=1+(u_sq/16384)*(4096+u_sq*(-768+u_sq*(320-175*u_sq)))
+        B=(u_sq/1024)*(256+u_sq*(-128+u_sq*(74-47*u_sq)))
+        delta_sig=B*sin_sigma*(cos2_sigma_m+0.25*B*(cos_sigma*(-1+2*cos2_sigma_m**2)-(1/6)*B*cos2_sigma_m*(-3+4*sin_sigma**2)*(-3+4*cos2_sigma_m**2)))
+
+        alpha12 = get_heading(coord1, coord2)
+        m=(b*A*(sigma-delta_sig))#/1000                # output distance in m
+        return (m,alpha12)
+    except ZeroDivisionError:
+        return (0,0)
+
+
+def direct(phi1, lembda1, alpha12, s): #lat, lon, bearing, distance
+    global a
+    global f
+    global b
+
+    piD4 = atan( 1.0 )
+    two_pi = piD4 * 8.0
+    phi1    = phi1    * piD4 / 45.0
+    lembda1 = lembda1 * piD4 / 45.0
+    alpha12 = alpha12 * piD4 / 45.0
+    if ( alpha12 < 0.0 ) :
+        alpha12 = alpha12 + two_pi
+    if ( alpha12 > two_pi ) :
+        alpha12 = alpha12 - two_pi
+    TanU1 = (1-f) * tan(phi1)
+    U1 = atan( TanU1 )
+    sigma1 = atan2( TanU1, cos(alpha12) )
+    Sinalpha = cos(U1) * sin(alpha12)
+    cosalpha_sq = 1.0 - Sinalpha * Sinalpha
+    u2 = cosalpha_sq * (a * a - b * b ) / (b * b)
+    A = 1.0 + (u2 / 16384) * (4096 + u2 * (-768 + u2 * \
+        (320 - 175 * u2) ) )
+    B = (u2 / 1024) * (256 + u2 * (-128 + u2 * (74 - 47 * u2) ) )
+    # Starting with the approx
+    sigma = (s / (b * A))
+    last_sigma = 2.0 * sigma + 2.0   # something impossible
+
+    # Iterate the following 3 eqs unitl no sig change in sigma
+    # two_sigma_m , delta_sigma
+    while ( abs( (last_sigma - sigma) / sigma) > 1.0e-9 ):
+        two_sigma_m = 2 * sigma1 + sigma
+        delta_sigma = B * sin(sigma) * ( cos(two_sigma_m) \
+                + (B/4) * (cos(sigma) * \
+                (-1 + 2 * pow( cos(two_sigma_m), 2 ) -  \
+                (B/6) * cos(two_sigma_m) * \
+                (-3 + 4 * pow(sin(sigma), 2 )) *  \
+                (-3 + 4 * pow( cos (two_sigma_m), 2 )))))
+        last_sigma = sigma
+        sigma = (s / (b * A)) + delta_sigma
+    phi2 = atan2 ( (sin(U1) * cos(sigma) +\
+        cos(U1) * sin(sigma) * cos(alpha12) ), \
+        ((1-f) * sqrt( pow(Sinalpha, 2) +  \
+        pow(sin(U1) * sin(sigma) - cos(U1) * \
+        cos(sigma) * cos(alpha12), 2))))
+    lembda = atan2( (sin(sigma) * sin(alpha12 )),\
+        (cos(U1) * cos(sigma) -  \
+        sin(U1) *  sin(sigma) * cos(alpha12)))
+    C = (f/16) * cosalpha_sq * (4 + f * (4 - 3 * cosalpha_sq ))
+    omega = lembda - (1-C) * f * Sinalpha *  \
+        (sigma + C * sin(sigma) * (cos(two_sigma_m) + \
+        C * cos(sigma) * (-1 + 2 *\
+        pow(cos(two_sigma_m), 2) )))
+    lembda2 = lembda1 + omega
+    alpha21 = atan2 ( Sinalpha, (-sin(U1) * \
+        sin(sigma) +
+        cos(U1) * cos(sigma) * cos(alpha12)))
+    alpha21 = alpha21 + two_pi / 2.0
+    if ( alpha21 < 0.0 ) :
+        alpha21 = alpha21 + two_pi
+    if ( alpha21 > two_pi ) :
+        alpha21 = alpha21 - two_pi
+    phi2 = phi2 * 45.0 / piD4
+    lembda2 = lembda2 * 45.0 / piD4
+    alpha21 = alpha21 * 45.0 / piD4
+    return (phi2, lembda2)#, alpha21
+
+if __name__ == '__main__':
+    help = """Usage:
+    vincenty.py [option] [value1] [value2] [value3] [value4]
+    Get distance between two points in meters:
+        vincenty.py inverse lat1 lon1 lat2 lon2
+    Get coordinate at a given distance and heading:
+        vincenty.py direct lat1 lon1 heading distance
+    Get the heading between two coordinates:
+        vincenty.py heading lat1 lon1 lat2 lon2"""
+    try:
+        op1 = float(sys.argv[2])
+        op2 = float(sys.argv[3])
+        op3 = float(sys.argv[4])
+        op4 = float(sys.argv[5])
+
+        if sys.argv[1] == "inverse":
+            output = inverse((op1, op2),(op3, op4))[0]
+        elif sys.argv[1] == "direct":
+            output = str(direct(op1, op2, op3, op4))[1:-1]
+        elif sys.argv[1] == "heading":
+            output = get_heading((op1, op2),(op3, op4))
+        else:
+            output = help
+        print(output)
+    except:
+        print(help)