122 lines
4 KiB
Python
122 lines
4 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Pre-warm the Redis tile cache by fetching all tiles for a bounding box.
|
||
|
||
Tiles are fetched through the backend, which caches them in Redis automatically.
|
||
Run this in the background after starting the stack:
|
||
|
||
nohup python3 scripts/seed_tiles.py &
|
||
|
||
Usage:
|
||
python3 seed_tiles.py [min_lon] [min_lat] [max_lon] [max_lat] [min_zoom] [max_zoom]
|
||
|
||
Defaults to the Netherlands at zoom levels 0-12.
|
||
"""
|
||
|
||
import math
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
import urllib.error
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
import os
|
||
|
||
BACKEND_URL = os.environ.get("BACKEND_URL", "http://localhost:8080")
|
||
LAYERS = ["planet_osm_polygon", "planet_osm_line", "planet_osm_point", "planet_osm_roads"]
|
||
WORKERS = int(os.environ.get("SEED_WORKERS", "4"))
|
||
|
||
|
||
def wait_for_backend():
|
||
"""Wait until the backend health endpoint responds."""
|
||
url = f"{BACKEND_URL}/health"
|
||
print(f"Waiting for backend at {url}...")
|
||
while True:
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=5) as resp:
|
||
if resp.status == 200:
|
||
print("Backend is up.")
|
||
return
|
||
except Exception:
|
||
pass
|
||
time.sleep(3)
|
||
|
||
|
||
def lon_to_x(lon, zoom):
|
||
return int((lon + 180.0) / 360.0 * (1 << zoom))
|
||
|
||
|
||
def lat_to_y(lat, zoom):
|
||
lat_r = math.radians(lat)
|
||
return int((1.0 - math.log(math.tan(lat_r) + 1.0 / math.cos(lat_r)) / math.pi) / 2.0 * (1 << zoom))
|
||
|
||
|
||
def tiles_for_bbox(min_lon, min_lat, max_lon, max_lat, zoom):
|
||
max_coord = (1 << zoom) - 1
|
||
x_min = max(0, min(lon_to_x(min_lon, zoom), max_coord))
|
||
x_max = max(0, min(lon_to_x(max_lon, zoom), max_coord))
|
||
y_min = max(0, min(lat_to_y(max_lat, zoom), max_coord)) # max_lat → smaller y
|
||
y_max = max(0, min(lat_to_y(min_lat, zoom), max_coord)) # min_lat → larger y
|
||
for x in range(x_min, x_max + 1):
|
||
for y in range(y_min, y_max + 1):
|
||
yield zoom, x, y
|
||
|
||
|
||
def fetch_tile(layer, z, x, y):
|
||
url = f"{BACKEND_URL}/tiles/{layer}/{z}/{x}/{y}.pbf"
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=120) as resp:
|
||
return resp.status == 200
|
||
except urllib.error.HTTPError:
|
||
return False
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def main():
|
||
wait_for_backend()
|
||
|
||
min_lon = float(sys.argv[1]) if len(sys.argv) > 1 else 3.3
|
||
min_lat = float(sys.argv[2]) if len(sys.argv) > 2 else 50.7
|
||
max_lon = float(sys.argv[3]) if len(sys.argv) > 3 else 7.2
|
||
max_lat = float(sys.argv[4]) if len(sys.argv) > 4 else 53.6
|
||
min_zoom = int(sys.argv[5]) if len(sys.argv) > 5 else 0
|
||
max_zoom = int(sys.argv[6]) if len(sys.argv) > 6 else 12
|
||
|
||
# Build full task list
|
||
tasks = []
|
||
for zoom in range(min_zoom, max_zoom + 1):
|
||
coords = list(tiles_for_bbox(min_lon, min_lat, max_lon, max_lat, zoom))
|
||
tile_count = len(coords)
|
||
print(f" z{zoom}: {tile_count} tiles × {len(LAYERS)} layers = {tile_count * len(LAYERS)} requests")
|
||
for z, x, y in coords:
|
||
for layer in LAYERS:
|
||
tasks.append((layer, z, x, y))
|
||
|
||
total = len(tasks)
|
||
print(f"\nTotal: {total} requests — starting with {WORKERS} workers...\n")
|
||
|
||
done = 0
|
||
errors = 0
|
||
start = time.time()
|
||
|
||
with ThreadPoolExecutor(max_workers=WORKERS) as executor:
|
||
futures = {executor.submit(fetch_tile, *t): t for t in tasks}
|
||
for future in as_completed(futures):
|
||
ok = future.result()
|
||
done += 1
|
||
if not ok:
|
||
errors += 1
|
||
if done % 50 == 0 or done == total:
|
||
elapsed = time.time() - start
|
||
rate = done / elapsed if elapsed > 0 else 0
|
||
eta = (total - done) / rate if rate > 0 else 0
|
||
pct = 100 * done // total
|
||
print(f" {done}/{total} ({pct}%) | {errors} errors | {rate:.1f} req/s | ETA {eta:.0f}s")
|
||
|
||
elapsed = time.time() - start
|
||
print(f"\nDone: {total - errors}/{total} tiles seeded in {elapsed:.0f}s")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|