diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 707323c..dcef475 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -91,24 +91,22 @@ services: ports: - "5002:5000" -importer: - image: docker.io/iboates/osm2pgsql:latest - networks: - - maps-net - volumes: - - ../data:/data - environment: - DATABASE_URL: "postgres://maps:maps@postgres:5432/maps" - command: > - osm2pgsql - --database postgres://maps:maps@postgres:5432/maps - --create --slim -G --hstore - /data/region.osm.pbf - depends_on: - - postgres - profiles: - - import - restart: "no" + importer: + build: + context: . + dockerfile: importer.Dockerfile + networks: + - maps-net + volumes: + - ../data:/data + environment: + PG_CONN: "postgres://maps:maps@postgres:5432/maps" + PBF_FILE: "/data/region.osm.pbf" + depends_on: + - postgres + profiles: + - import + restart: "no" volumes: maps-pgdata: diff --git a/backend/importer.Dockerfile b/backend/importer.Dockerfile new file mode 100644 index 0000000..db089fa --- /dev/null +++ b/backend/importer.Dockerfile @@ -0,0 +1,18 @@ +FROM docker.io/arm64v8/debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + osm2pgsql \ + postgresql-client \ + wget \ + git \ + lua5.3 \ + curl \ + ca-certificates \ + redis-tools \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY scripts/ /app/scripts/ +RUN chmod +x /app/scripts/*.sh + +ENTRYPOINT ["/bin/bash"] diff --git a/backend/scripts/01_download.sh b/backend/scripts/01_download.sh new file mode 100755 index 0000000..7bac223 --- /dev/null +++ b/backend/scripts/01_download.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# scripts/01_download.sh + + +REGION="europe/netherlands" +DATA_DIR="/data/osm" +GEOFABRIK_BASE="https://download.geofabrik.de" + +mkdir -p "$DATA_DIR" + +# Download PBF extract (or update if already present) +wget -N "${GEOFABRIK_BASE}/${REGION}-latest.osm.pbf" \ + -O "${DATA_DIR}/region.osm.pbf" + +# Download the corresponding state file for future diff updates +wget -N "${GEOFABRIK_BASE}/${REGION}-updates/state.txt" \ + -O "${DATA_DIR}/state.txt" diff --git a/backend/scripts/02_import_tiles.sh b/backend/scripts/02_import_tiles.sh new file mode 100755 index 0000000..f36dad3 --- /dev/null +++ b/backend/scripts/02_import_tiles.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# scripts/02_import_tiles.sh + +PBF_FILE="/data/osm/region.osm.pbf" +PG_CONN="postgresql://maps:maps@postgres:5432/maps" + +# Clone openmaptiles toolchain (once) +if [ ! -d "/opt/openmaptiles" ]; then + git clone https://github.com/openmaptiles/openmaptiles.git /opt/openmaptiles +fi + +# Import OSM data into PostGIS using openmaptiles schema +# This creates the tables that Martin reads for tile generation +cd /opt/openmaptiles + +# osm2pgsql import with openmaptiles mapping +osm2pgsql \ + --create \ + --slim \ + --database "$PG_CONN" \ + --style openmaptiles.style \ + --tag-transform-script lua/tagtransform.lua \ + --number-processes 4 \ + --cache 4096 \ + --flat-nodes /data/osm/nodes.cache \ + "$PBF_FILE" + +# Run openmaptiles SQL post-processing to create materialized views +# that Martin serves as tile layers +psql "$PG_CONN" -f build/openmaptiles.sql + +echo "Tile data import complete. Martin will serve tiles from PostGIS." diff --git a/backend/scripts/03_import_pois.sh b/backend/scripts/03_import_pois.sh new file mode 100755 index 0000000..6473060 --- /dev/null +++ b/backend/scripts/03_import_pois.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# scripts/03_import_pois.sh + +PBF_FILE="/data/osm/region.osm.pbf" +PG_CONN="postgresql://maps:maps@postgres:5432/maps" + +# Run the initial migration to create the pois table +psql "$PG_CONN" -f /app/migrations/001_create_pois.sql + +# Import POIs using osm2pgsql with a custom Lua transform +osm2pgsql \ + --create \ + --output=flex \ + --style /app/scripts/poi_flex.lua \ + --database "$PG_CONN" \ + --cache 2048 \ + --number-processes 4 \ + --flat-nodes /data/osm/nodes.cache \ + "$PBF_FILE" + +echo "POI import complete." diff --git a/backend/scripts/04_import_geocoding.sh b/backend/scripts/04_import_geocoding.sh new file mode 100755 index 0000000..b635814 --- /dev/null +++ b/backend/scripts/04_import_geocoding.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# scripts/04_import_geocoding.sh + +PBF_FILE="/data/osm/region.osm.pbf" +NOMINATIM_DATA="/data/nominatim" +PHOTON_DATA="/data/photon" + +# --- Nominatim Import --- +# Nominatim builds a PostgreSQL database with geocoding data. +# Photon reads from this database to build its Elasticsearch index. + +nominatim import \ + --osm-file "$PBF_FILE" \ + --project-dir "$NOMINATIM_DATA" \ + --threads 4 + +# --- Photon Import --- +# Photon reads the Nominatim database and builds an Elasticsearch index. +# This index is what Photon uses to serve search queries. + +java -jar /opt/photon/photon.jar \ + -nominatim-import \ + -host localhost \ + -port 5432 \ + -database nominatim \ + -user nominatim \ + -password nominatim \ + -data-dir "$PHOTON_DATA" \ + -languages en,nl,de,fr + +echo "Geocoding index built. Photon is ready to serve." diff --git a/backend/scripts/05_import_routing.sh b/backend/scripts/05_import_routing.sh new file mode 100755 index 0000000..677847c --- /dev/null +++ b/backend/scripts/05_import_routing.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# scripts/05_import_routing.sh + +PBF_FILE="/data/osm/region.osm.pbf" +OSRM_DATA="/data/osrm" + +# Process each profile: driving, walking, cycling +for PROFILE in car foot bicycle; do + PROFILE_DIR="${OSRM_DATA}/${PROFILE}" + mkdir -p "$PROFILE_DIR" + cp "$PBF_FILE" "${PROFILE_DIR}/region.osm.pbf" + + # Step 1: Extract — parse the PBF and produce an .osrm file + # Uses the appropriate profile from OSRM's bundled profiles + osrm-extract \ + --profile /opt/osrm-profiles/${PROFILE}.lua \ + --threads 4 \ + "${PROFILE_DIR}/region.osm.pbf" + + # Step 2: Partition — create a recursive multi-level partition + osrm-partition \ + "${PROFILE_DIR}/region.osrm" + + # Step 3: Customize — compute edge weights for the partition + osrm-customize \ + "${PROFILE_DIR}/region.osrm" + + echo "OSRM ${PROFILE} profile ready." +done + +echo "All OSRM profiles processed." diff --git a/backend/scripts/06_build_offline_packages.sh b/backend/scripts/06_build_offline_packages.sh new file mode 100755 index 0000000..34085ed --- /dev/null +++ b/backend/scripts/06_build_offline_packages.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# scripts/06_build_offline_packages.sh + +PG_CONN="postgresql://maps:maps@postgres:5432/maps" +PACKAGES_DIR="/data/offline_packages" +REGION_ID="amsterdam" +BBOX="4.7288,52.2783,5.0796,52.4311" # minLon,minLat,maxLon,maxLat + +mkdir -p "${PACKAGES_DIR}/${REGION_ID}" + +# --- Tiles: extract MBTiles for the bounding box --- +# Use martin-cp (Martin's CLI tool) to export tiles from PostGIS to MBTiles +martin-cp \ + --output-file "${PACKAGES_DIR}/${REGION_ID}/tiles.mbtiles" \ + --mbtiles-type flat \ + --bbox "$BBOX" \ + --min-zoom 0 \ + --max-zoom 16 \ + --source openmaptiles \ + --connect "$PG_CONN" + +# --- POIs: export to SQLite with FTS5 index --- +# Custom Rust tool or Python script that queries PostGIS and writes SQLite +/app/tools/export_pois_sqlite \ + --bbox "$BBOX" \ + --pg-conn "$PG_CONN" \ + --output "${PACKAGES_DIR}/${REGION_ID}/pois.db" + +# --- Routing: tar the OSRM files per profile --- +for PROFILE in car foot bicycle; do + tar -cf "${PACKAGES_DIR}/${REGION_ID}/routing-${PROFILE}.tar" \ + -C "/data/osrm/${PROFILE}" \ + region.osrm region.osrm.cell_metrics region.osrm.cells \ + region.osrm.datasource_names region.osrm.ebg region.osrm.ebg_nodes \ + region.osrm.edges region.osrm.fileIndex region.osrm.geometry \ + region.osrm.icd region.osrm.maneuver_overrides \ + region.osrm.mldgr region.osrm.names region.osrm.nbg_nodes \ + region.osrm.partition region.osrm.properties \ + region.osrm.ramIndex region.osrm.timestamp \ + region.osrm.tld region.osrm.tls region.osrm.turn_duration_penalties \ + region.osrm.turn_penalties_index region.osrm.turn_weight_penalties +done + +# --- Update offline_regions table with file sizes --- +TILES_SIZE=$(stat -f%z "${PACKAGES_DIR}/${REGION_ID}/tiles.mbtiles" 2>/dev/null || stat -c%s "${PACKAGES_DIR}/${REGION_ID}/tiles.mbtiles") +ROUTING_SIZE=0 +for PROFILE in car foot bicycle; do + SIZE=$(stat -f%z "${PACKAGES_DIR}/${REGION_ID}/routing-${PROFILE}.tar" 2>/dev/null || stat -c%s "${PACKAGES_DIR}/${REGION_ID}/routing-${PROFILE}.tar") + ROUTING_SIZE=$((ROUTING_SIZE + SIZE)) +done +POIS_SIZE=$(stat -f%z "${PACKAGES_DIR}/${REGION_ID}/pois.db" 2>/dev/null || stat -c%s "${PACKAGES_DIR}/${REGION_ID}/pois.db") + +psql "$PG_CONN" < >(tee -a "$LOGFILE") 2>&1 + +echo "=== OSM data update started at $(date -u) ===" + +# Step 1: Download latest PBF +/app/scripts/01_download.sh + +# Step 2: Import tile data +/app/scripts/02_import_tiles.sh + +# Step 3: Import POI data +/app/scripts/03_import_pois.sh + +# Step 4: Update geocoding index +/app/scripts/04_import_geocoding.sh + +# Step 5: Rebuild OSRM routing graphs +/app/scripts/05_import_routing.sh + +# Step 6: Rebuild offline packages +/app/scripts/06_build_offline_packages.sh + +# Step 7: Flush tile cache in Redis (tiles have changed) +redis-cli -h redis FLUSHDB + +# Step 8: Restart services to pick up new data +docker compose restart martin osrm-driving osrm-walking osrm-cycling + +echo "=== OSM data update completed at $(date -u) ==="