Working on solving performance issues for large datasets

.gitignore

@@ -7,3 +7,4 @@ test*
 *.db
 *.txt
 *.czml
+fil-result/

df-aggregator.py

@@ -13,13 +13,15 @@ from colorsys import hsv_to_rgb
 from optparse import OptionParser
 from os import system, name, kill, getpid
 from lxml import etree
-from sklearn.cluster import DBSCAN
+from sklearn.cluster import DBSCAN, OPTICS
 from sklearn.preprocessing import StandardScaler, minmax_scale
 from geojson import Point, MultiPoint, Feature, FeatureCollection
 from czml3 import Packet, Document, Preamble
 from czml3.properties import Position, Polyline, PolylineOutlineMaterial, Color, Material
 from bottle import route, run, request, get, post, put, response, redirect, template, static_file
 
+import sys
+
 d = 40000 #draw distance of LOBs in meters
 max_age = 5000
 receivers = []
@@ -201,7 +203,12 @@ def process_data(database_name, outfile):
     try:
         # c.execute("SELECT COUNT(*) FROM intersects")
         # n_intersects = int(c.fetchone()[0])
-        c.execute("SELECT longitude, latitude, time FROM intersects")
+        # 200000 eats 22GB of RAM
+        # 187500 eats 13GB of RAM
+        # 175000 eats 1661 MB of RAM
+        # 150000 eats 976 MB of RAM
+        # 140000 eats 920 MB of RAM
+        c.execute("SELECT longitude, latitude, time FROM intersects LIMIT 120000")
         intersect_array = np.array(c.fetchall())
     except sqlite3.OperationalError:
         n_intersects = 0
@@ -214,12 +221,24 @@ def process_data(database_name, outfile):
 
     if intersect_array.size != 0:
         if ms.eps > 0:
-            X = StandardScaler().fit_transform(intersect_array[:,0:2])
+            X = StandardScaler().fit_transform(intersect_array[:,0:2:])
+            # X = np.radians(intersect_array[:,0:2])
 
+            n_points = len(X)
+            size_x = sys.getsizeof(X)/1024
+            print(f"The dataset is {size_x} kilobytes")
+            print(f"Computing Clusters from {n_points} intersections.")
             # Compute DBSCAN
+            # starttime = time.time()
+            # db = OPTICS(max_eps=ms.eps, min_samples=int(ms.min_samp), cluster_method="dbscan").fit(X)
+            # stoptime = time.time()
+            # print(f"OPTICS took {stoptime - starttime} seconds to compute the clusters.")
+
+            starttime = time.time()
+            # , algorithm='ball_tree', metric='haversine'
             db = DBSCAN(eps=ms.eps, min_samples=ms.min_samp).fit(X)
-            core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
-            core_samples_mask[db.core_sample_indices_] = True
+            stoptime = time.time()
+            print(f"DBSCAN took {stoptime - starttime} seconds to compute the clusters.")
             labels = db.labels_
 
             intersect_array = np.column_stack((intersect_array, labels))
@@ -250,6 +269,8 @@ def process_data(database_name, outfile):
                 ell_radius_y = np.sqrt(1 - pearson) * np.sqrt(c) * n_std
                 axis_x = v.inverse(Reverse(clustermean.tolist()), (ell_radius_x + clustermean[1], clustermean[0]))[0]
                 axis_y = v.inverse(Reverse(clustermean.tolist()), (clustermean[1], ell_radius_y + clustermean[0]))[0]
+                # axis_x = v.inverse(clustermean.tolist(), (ell_radius_x + clustermean[1], clustermean[0]))[0]
+                # axis_y = v.inverse(clustermean.tolist(), (clustermean[1], ell_radius_y + clustermean[0]))[0]
 
                 if b == 0 and a >= c:
                     rotation = 0
@@ -355,41 +376,12 @@ def write_czml(best_point, all_the_points, ellipsedata):
                 }
             }
         }
-    area_of_interest_properties = {
-        "granularity": 0.008722222,
-        "height": 0,
-        "material": {
-            "solidColor": {
-                "color": {
-                    "rgba": [0, 0, 255, 25]
-                    }
-                }
-            },
-        "outline": True,
-        "outlineWidth": 2,
-        "outlineColor": {"rgba": [53, 184, 240, 255],},
-        },
-
-    exclusion_area_properties = {
-        "granularity": 0.008722222,
-        "height": 0,
-        "material": {
-            "solidColor": {
-                "color": {
-                    "rgba": [242, 10, 0, 25]
-                    }
-                }
-            },
-        "outline": True,
-        "outlineWidth": 2,
-        "outlineColor": {"rgba": [224, 142, 0, 255],},
-        },
 
     top = Preamble(name="Geolocation Data")
     all_point_packets = []
     best_point_packets = []
     ellipse_packets = []
-    aoi_packets = []
+
     # exclusion_packets = []
 
     if len(all_the_points) > 0 and (ms.plotintersects or ms.eps == 0):
@@ -430,25 +422,7 @@ def write_czml(best_point, all_the_points, ellipsedata):
             ellipse={**ellipse_properties, **ellipse_info},
             position={"cartographicDegrees": [ x[3], x[4], 0 ]}))
 
-    for x in fetch_aoi_data():
-        aoi = {
-            'uid': x[0],
-            'aoi_type': x[1],
-            'latitude': x[2],
-            'longitude': x[3],
-            'radius': x[4]
-        }
-        if aoi['aoi_type'] == "aoi":
-            aoi_properties = area_of_interest_properties[0]
-        elif aoi['aoi_type'] == "exclusion":
-            aoi_properties = exclusion_area_properties[0]
-        aoi_info = {"semiMajorAxis": aoi['radius'], "semiMinorAxis": aoi['radius'], "rotation": 0}
-        aoi_packets.append(Packet(id=aoi['aoi_type'] + str(aoi['uid']),
-        ellipse={**aoi_properties, **aoi_info},
-        position={"cartographicDegrees": [ aoi['longitude'], aoi['latitude'], 0 ]}))
-
-    output = Document([top] + best_point_packets + all_point_packets + ellipse_packets +
-        aoi_packets)
+    output = Document([top] + best_point_packets + all_point_packets + ellipse_packets)
 
     return output
 
@@ -459,6 +433,7 @@ def write_rx_czml():
     height = 50
     receiver_point_packets = []
     lob_packets = []
+    aoi_packets = []
     top = Preamble(name="Receivers")
 
     rx_properties = {
@@ -469,6 +444,36 @@ def write_rx_czml():
         "width": 48,
     }
 
+    area_of_interest_properties = {
+        "granularity": 0.008722222,
+        "height": 0,
+        "material": {
+            "solidColor": {
+                "color": {
+                    "rgba": [0, 0, 255, 25]
+                    }
+                }
+            },
+        "outline": True,
+        "outlineWidth": 2,
+        "outlineColor": {"rgba": [53, 184, 240, 255],},
+        },
+
+    exclusion_area_properties = {
+        "granularity": 0.008722222,
+        "height": 0,
+        "material": {
+            "solidColor": {
+                "color": {
+                    "rgba": [242, 10, 0, 25]
+                    }
+                }
+            },
+        "outline": True,
+        "outlineWidth": 2,
+        "outlineColor": {"rgba": [224, 142, 0, 255],},
+        },
+
     for index, x in enumerate(receivers):
         if x.isActive and ms.receiving:
             lob_start_lat = x.latitude
@@ -501,7 +506,24 @@ def write_rx_czml():
         billboard={**rx_properties, **rx_icon},
         position={"cartographicDegrees": [ x.longitude, x.latitude, 15 ]}))
 
-    output = Document([top] + receiver_point_packets + lob_packets)
+    for x in fetch_aoi_data():
+        aoi = {
+            'uid': x[0],
+            'aoi_type': x[1],
+            'latitude': x[2],
+            'longitude': x[3],
+            'radius': x[4]
+        }
+        if aoi['aoi_type'] == "aoi":
+            aoi_properties = area_of_interest_properties[0]
+        elif aoi['aoi_type'] == "exclusion":
+            aoi_properties = exclusion_area_properties[0]
+        aoi_info = {"semiMajorAxis": aoi['radius'], "semiMinorAxis": aoi['radius'], "rotation": 0}
+        aoi_packets.append(Packet(id=aoi['aoi_type'] + str(aoi['uid']),
+        ellipse={**aoi_properties, **aoi_info},
+        position={"cartographicDegrees": [ aoi['longitude'], aoi['latitude'], 0 ]}))
+
+    output = Document([top] + receiver_point_packets + lob_packets + aoi_packets)
 
     return output
 
@@ -795,9 +817,7 @@ def run_receiver(receivers):
                         lon_rxb = previous[1]
                         conf_rxb = previous[2]
                         doa_rxb = previous[3]
-                        # if abs(doa_rxa - doa_rxb) > 5:
                         spacial_diversity, z = v.inverse((lat_rxa, lon_rxa), (lat_rxb, lon_rxb))
-                        #print(f"Distance from other points: {spacial_diversity}")
                         min_diversity = 500
                         if (spacial_diversity > min_diversity and
                              abs(doa_rxa - doa_rxb) > 5):
@@ -817,7 +837,6 @@ def run_receiver(receivers):
                 print("Problem connecting to receiver.")
                 # ms.receiving = False
 
-
         time.sleep(1)
         if dots > 5:
             dots = 1

static/receiver_configurator.js

 // Update Map every n milliseconds
 var refreshrate = 600000;
-var autoRefresh = setInterval(function () { updateParams(); }, refreshrate);
+// var autoRefresh = setInterval(function () { updateParams(); }, refreshrate);
 
 // *************************************************
 // * Gets Rx data from backend
@@ -64,7 +64,7 @@ function editReceivers(rx_json, id) {
     var isSingleCheck;
     var editButton = document.getElementById(id + "-edit");
     if (editButton.checked) {
-        clearInterval(autoRefresh);
+        // clearInterval(autoRefresh);
         let isMobile = "";
         if (receivers[id].mobile) isMobile = "checked";
         let isInverted = "";
@@ -106,7 +106,7 @@ function editReceivers(rx_json, id) {
         //     }
         // }
     } else {
-        autoRefresh = setInterval(function () { updateParams(); }, refreshrate);
+        // autoRefresh = setInterval(function () { updateParams(); }, refreshrate);
         isMobileCheck = document.getElementById("mobilerx_toggle_" + id);
         if (isMobileCheck.checked) {
             receivers[id].mobile = true;

views/cesium.tpl

@@ -27,7 +27,7 @@
     var transmittersDataSource;
     var receiversDataSource;
     // Your access token can be found at: https://cesium.com/ion/tokens.
-    Cesium.Ion.defaultAccessToken = '{{access_token}}';
+    // Cesium.Ion.defaultAccessToken = '{{access_token}}';
     // var hpr = new Cesium.HeadingPitchRange(0, 40, 0)
     var viewer = new Cesium.Viewer('cesiumContainer', {
       // terrainProvider: Cesium.createWorldTerrain(),