Merge branch 'natural_earth_maps' into main

Map.cpp

@@ -4,6 +4,16 @@
 
 bool Map::QTInsert(QuadTree *tree, Line *line, int depth) {
 
+  // if(depth > 25) {
+  //  // printf("fail [%f %f] -> [%f %f]\n",line->start.lon,line->start.lat,line->end.lon,line->end.lat);
+
+  //  // printf("bounds %f %f %f %f\n",tree->lon_min, tree->lon_max, tree->lat_min, tree->lat_max);
+  //   // fflush(stdout); 
+  //    tree->lines.push_back(&(*line));
+  //    return true;
+  // }
+  
+
   bool startInside = line->start.lat >= tree->lat_min &&
    line->start.lat <= tree->lat_max &&
    line->start.lon >= tree->lon_min &&
@@ -14,6 +24,9 @@ bool Map::QTInsert(QuadTree *tree, Line *line, int depth) {
    line->end.lon >= tree->lon_min &&
    line->end.lon <= tree->lon_max;
 
+  // if (!startInside || !endInside) {
+  //   return false; 
+  // }
   if (!startInside && !endInside) {
     return false; 
   }
@@ -76,6 +89,7 @@ bool Map::QTInsert(QuadTree *tree, Line *line, int depth) {
 	} 
 	
   tree->lines.push_back(&(*line));
+
   return true;
 }
 
@@ -178,6 +192,8 @@ Map::Map() {
 		} 
 	}
 
+  printf("map bounds: %f %f %f %f\n",root.lon_min, root.lon_max, root.lat_min, root.lat_max);
+
   Point currentPoint;
   Point nextPoint;
 
@@ -196,7 +212,10 @@ Map::Map() {
     nextPoint.lon = mapPoints[i + 2];
     nextPoint.lat = mapPoints[i + 3];
 
+    // printf("inserting [%f %f] -> [%f %f]\n",currentPoint.lon,currentPoint.lat,nextPoint.lon,nextPoint.lat);
+
     QTInsert(&root, new Line(currentPoint, nextPoint), 0);
 	}
 
+  printf("done\n");
 }

README.md

@@ -1,12 +1,20 @@
 # viz1090
 
 **This is work in progress**
-There are a lot of missing pieces in this implementation so far:
+
-* A proper map system yet. Eventually map data should be pulled from Mapbox or similar.
+
-* In-application menus or configuration yet.
+There are some major fixes and cleanup that need to happen before a relase:
+* Everything is a grab bag of C and C++, need to more consistently modernize
+* A full refactor, especially View.cpp, necessary for many of the new features below.
+* A working Android build, as this is the best way to run this on portable hardware.
+
+There are also a lot of missing features:
+* Map improvements
+	* Labels, different colors/line weights for features
+	* Tile prerenderer for improved performance
+* In-application menus for view options and configuration
 * Theming/colormaps (important as this is primarily intended to be eye candy!)
 * Integration with handheld features like GPS, battery monitors, buttons/dials, etc. 
-* Android build is currently broken
 
 ### BUILDING
 
@@ -34,30 +42,21 @@ make clean; make
 ```
 
 3. Download and process map data
-Until more comprehensive map source (e.g., Mapbox) is integrated, viz1090 uses the lat/lon SVG files from https://www.mccurley.org
+Grab a shapefile with your desired level of detail from https://www.naturalearthdata.com/downloads
 
-The getmap.sh pulls the large SVG file for the contiguous 48 US states and produces a binary file for viz1090 to read.
+[This](https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces.zip) is a good place to start.
+
+Unzip and copy the .shp and .shx files. 
 
 ```
 sudo apt install python3 python3-pip
-pip3 install lxml numpy tqdm
+pip3 install geopandas tqdm
-./getmap.sh
+python3 mapconverter.py ne_10m_admin_1_states_provinces.shp
 ```
 
-There is also a world map avaiable from McCurley (https://mccurley.org/svg/data/World.svgz), which is much lower resolution and thus better for lower power hardware.
+This will produce a file mapdata.bin that viz1090 reads. If the file doesn't exist then visualizer will show planes and trails without any geography.
 
-The mapconverter script called by getmap.sh downsamples the file to render resonably quickly on a Raspberri Pi 4. If you are on a slower device (e.g, a Raspberry Pi 3), you may want to try something like:
+The default parameters for mapconverter should render resonably quickly on a Raspberri Pi 4. See the mapconverter section below for other options.
-
-```
-python3 mapconverter.py --resolution 64 all.svg
-```
-
-On the other hand, if you are on a modern desktop or laptop, you can use something higher (but you probably don't need the full 6 digit precision of the McCurley SVG file):
-
-
-```
-python3 mapconverter.py --resolution 8192 all.svg
-```
 
 
 3. (Windows only)

View.cpp

@@ -292,7 +292,7 @@ void View::font_init() {
     // todo separate style stuff
     //
 
-    SDL_Color bgcolor = {0,0,0,255};
+    SDL_Color bgcolor = {0,0,20,255};
     SDL_Color greenblue = {236,192,68,255};
     SDL_Color lightblue = {211,208,203,255};
     SDL_Color mediumblue ={110,136,152,255};
@@ -667,6 +667,33 @@ void View::drawLinesRecursive(QuadTree *tree, float screen_lat_min, float screen
         lineRGBA(renderer, x1, y1, x2, y2, style.mapInnerColor.r, style.mapInnerColor.g, style.mapInnerColor.b, 255);     
     }
 
+    // //Debug quadtree
+    // int tl_x,tl_y,tr_x,tr_y,bl_x,bl_y,br_x,br_y;
+    // float dx,dy;
+
+    // pxFromLonLat(&dx, &dy, tree->lon_min, tree->lat_min); 
+    // screenCoords(&tl_x, &tl_y, dx, dy);
+
+    // pxFromLonLat(&dx, &dy, tree->lon_max, tree->lat_min); 
+    // screenCoords(&tr_x, &tr_y, dx, dy);
+
+    // pxFromLonLat(&dx, &dy, tree->lon_min, tree->lat_max); 
+    // screenCoords(&bl_x, &bl_y, dx, dy);
+
+    // pxFromLonLat(&dx, &dy, tree->lon_max, tree->lat_max); 
+    // screenCoords(&br_x, &br_y, dx, dy);    
+
+    // lineRGBA(renderer, tl_x, tl_y, tr_x, tr_y, 50, 50, 50, 255);     
+    // lineRGBA(renderer, tr_x, tr_y, br_x, br_y, 50, 50, 50, 255);     
+    // lineRGBA(renderer, bl_x, bl_y, br_x, br_y, 50, 50, 50, 255);     
+    // lineRGBA(renderer, tl_x, tl_y, bl_x, bl_y, 50, 50, 50, 255);  
+
+    // // pixelRGBA(renderer,tl_x, tl_y,255,0,0,255);
+    // pixelRGBA(renderer,tr_x, tr_y,0,255,0,255);
+    // pixelRGBA(renderer,bl_x, bl_y,0,0,255,255);
+    // pixelRGBA(renderer,br_x, br_y,255,255,0,255);
+
+   
 }
 
 

getmap.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-wget -O all.svg.gz https://www.mccurley.org/svg/data/allzips.svgz
+wget https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces.zip
-gunzip all.svg.gz
+unzip ne_10m_admin_1_states_provinces.zip
-python3 mapconverter.py all.svg
+python3 mapconverter.py ne_10m_admin_1_states_provinces.shp

mapconverter-cartopy.py

@@ -0,0 +1,64 @@
+import cartopy.io.shapereader as shpreader
+import numpy as np
+from tqdm import tqdm 
+import zipfile
+from io import BytesIO
+#from urllib.request import urlopen
+import requests
+import argparse
+import os
+
+def convertLinestring(linestring):
+	outlist = []
+
+	pointx = linestring.coords.xy[0]
+	pointy = linestring.coords.xy[1]
+			
+	for j in range(len(pointx)):
+		outlist.extend([float(pointx[j]),float(pointy[j])])
+
+	outlist.extend([0,0])
+	return outlist
+
+def extractLines(shapefile, tolerance):
+	print("Extracting map lines")
+	outlist = []
+
+	for i in tqdm(range(len(shapefile))):
+
+		simplified = shapefile[i].geometry.simplify(tolerance, preserve_topology=False)
+
+		if(simplified.geom_type == "LineString"):
+			outlist.extend(convertLinestring(simplified))
+			
+		elif(simplified.geom_type == "MultiPolygon" or simplified.geom_type == "Polygon"):
+
+			if(simplified.boundary.geom_type == "MultiLineString"):
+				for boundary in simplified.boundary:
+					outlist.extend(convertLinestring(boundary))
+			else:
+				outlist.extend(convertLinestring(simplified.boundary))
+	
+		else:
+			print("Unsupported type: " + simplified.geom_type)
+
+
+
+	return outlist
+
+parser = argparse.ArgumentParser(description='viz1090 Natural Earth Data Map Converter')
+parser.add_argument("--tolerance", default=0.001, type=float, help="map simplification tolerance")
+parser.add_argument("--scale", default="10m", choices=["10m","50m","110m"], type=str, help="map file scale")
+parser.add_argument("mapfile", type=str, help="shapefile to load (e.g., from https://www.naturalearthdata.com/downloads/")	
+
+args = parser.parse_args()
+
+shapefile = list(shpreader.Reader(args.mapfile).records())
+
+outlist = extractLines(shapefile, args.tolerance)
+
+bin_file = open("mapdata.bin", "wb")
+np.asarray(outlist).astype(np.single).tofile(bin_file)
+bin_file.close()
+
+print("Wrote %d points" % (len(outlist) / 2))

mapconverter.py

@@ -1,64 +1,61 @@
-from lxml import etree as ElementTree
+import geopandas
 import numpy as np
-import sys
 from tqdm import tqdm 
+import zipfile
+from io import BytesIO
+#from urllib.request import urlopen
+import requests
 import argparse
+import os
 
-parser = argparse.ArgumentParser(description='viz1090 SVG Map Converter')
+def convertLinestring(linestring):
-parser.add_argument("--resolution", default=250, type=int, nargs=1, help="downsample resolution")
+	outlist = []
-parser.add_argument("file", nargs=1, help="filename")
+
+	pointx = linestring.coords.xy[0]
+	pointy = linestring.coords.xy[1]
+			
+	for j in range(len(pointx)):
+		outlist.extend([float(pointx[j]),float(pointy[j])])
+
+	outlist.extend([0,0])
+	return outlist
+
+def extractLines(shapefile, tolerance):
+	print("Extracting map lines")
+	outlist = []
+	simplified = shapefile['geometry'].simplify(tolerance, preserve_topology=False)
+
+	for i in tqdm(range(len(simplified))):
+		if(simplified[i].geom_type == "LineString"):
+			outlist.extend(convertLinestring(simplified[i]))
+			
+		elif(simplified[i].geom_type == "MultiPolygon" or simplified[i].geom_type == "Polygon"):
+
+			if(simplified[i].boundary.geom_type == "MultiLineString"):
+				for boundary in simplified[i].boundary:
+					outlist.extend(convertLinestring(boundary))
+			else:
+				outlist.extend(convertLinestring(simplified[i].boundary))
+	
+		else:
+			print("Unsupported type: " + simplified[i].geom_type)
+
+
+
+	return outlist
+
+parser = argparse.ArgumentParser(description='viz1090 Natural Earth Data Map Converter')
+parser.add_argument("--tolerance", default=0.001, type=float, help="map simplification tolerance")
+parser.add_argument("--scale", default="10m", choices=["10m","50m","110m"], type=str, help="map file scale")
+parser.add_argument("mapfile", type=str, help="shapefile to load (e.g., from https://www.naturalearthdata.com/downloads/")	
 
 args = parser.parse_args()
 
+shapefile = geopandas.read_file(args.mapfile)
 
-if(len(args.file) == 0):
+outlist = extractLines(shapefile, args.tolerance)
-	print("No input filename given")
-	exit()
-
-parser = ElementTree.XMLParser(recover=True)
-tree = ElementTree.parse(args.file[0], parser)
-polys = tree.xpath('//polygon')
 
 bin_file = open("mapdata.bin", "wb")
-
-outlist = []
-
-
-resolution = args.resolution[0]
-
-print("Reading points")
-for i in tqdm(range(len(polys))):
-#for i in range(40):
-	p = polys[i]
-	currentPoints = (p.attrib['points']).replace(","," ").split()
-
-	if(len(currentPoints) == 14): #remove little circles in the McCurley maps
-		continue
-
-	prevx = 0
-	prevy = 0
-
-	temp = []
-
-	for i in range(int(len(currentPoints)/2)):
-		#currentPoints[2 * i + 0] = "%.*f" % (precision, float(currentPoints[2 * i + 0]))
-                #currentPoints[2 * i + 1] = "%.*f" % (precision, float(currentPoints[2 * i + 1]))
-
-            currentPoints[2 * i + 0] = float(int(resolution * float(currentPoints[2 * i + 0]))) / resolution
-            currentPoints[2 * i + 1] = float(int(resolution * float(currentPoints[2 * i + 1]))) / resolution
-
-            if(currentPoints[2 * i + 0] != prevx or currentPoints[2 * i + 1] != prevy):
-                temp.extend([currentPoints[2 * i + 0],currentPoints[2 * i + 1]])
-
-            prevx = currentPoints[2 * i + 0]
-            prevy = currentPoints[2 * i + 1] 
-
-	if(len(currentPoints) > 6): #must be at least a triangle
-                outlist.extend(temp)
-                #outlist.extend([temp[0],temp[1]])
-                outlist.extend(["0","0"])
-
-
 np.asarray(outlist).astype(np.single).tofile(bin_file)
 bin_file.close()