From b2d33012c588283d54662ab79a4ba5a0bf776010 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Wed, 23 Jul 2025 14:20:04 +0200 Subject: [PATCH 01/41] [CI] add CI pipeline as in github --- Jenkinsfile | 55 +++++++++++++++++++++++++++++++++++++++++++ config/so.test | 14 +++++++++++ launch_ci.sh | 21 +++++++++++++++++ podman-compose.ci.yml | 26 ++++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 Jenkinsfile create mode 100644 config/so.test create mode 100755 launch_ci.sh create mode 100644 podman-compose.ci.yml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..b19099134 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,55 @@ +pipeline { + agent { + label 'ansible-agent' + } + + parameters { + // Define boolean variables for project deployment - user input + booleanParam(name: 'LaunchCI', defaultValue: true, description: 'Launch CI tests') + } + + environment { + // Static vars + REPO_SO_BASE_URL = 'git@git.smart-origin.com:SmartOrigin' + REPO_BACKEND_NAME = 'c2c_v6_api' + WORKDIR = 'tmp_c2c_ci' + PODMAN_ARGS = '-p c2c_ci -f podman-compose.ci.yml' + } + + stages { + stage('Launch CI tests') { + //The when block control is this stage should be run base on the params given + when { + expression { + params.LaunchCI == true + } + } + steps { + script { + sh """ + pwd + podman-compose ${env.PODMAN_ARGS} up -d + sleep 60 + """ + try { sh "podman-compose ${env.PODMAN_ARGS} exec test /c2c_ci/launch_ci.sh" } + catch (Exception e) { + currentBuild.result = 'FAILURE' // Mark the build as failed + error "CI failed: ${e.getMessage()}" + } + sh "podman-compose ${env.PODMAN_ARGS} down" + } + } + } + } + post { + // Your post-build actions here + failure { + // Actions to take when any step or stage fails + echo 'The build has failed. Take a look at the logs and try again' + } + success { + // Actions to take when any step or stage fails + echo 'The build has succeded. Enjoy!' + } + } +} diff --git a/config/so.test b/config/so.test new file mode 100644 index 000000000..25e56af3e --- /dev/null +++ b/config/so.test @@ -0,0 +1,14 @@ +#!/bin/sh + +instanceid="github" +base_url="/${instanceid}" +db_name="c2corg_tests" +tests_db_host=postgresql +tests_db_name="c2corg_tests" +elasticsearch_port=9200 +elasticsearch_index="c2corg_tests" +tests_elasticsearch_host=elasticsearch +tests_elasticsearch_port=9200 +tests_elasticsearch_index="c2corg_tests" +redis_url="redis://redis:6379/" +version=0.0.0dev0 diff --git a/launch_ci.sh b/launch_ci.sh new file mode 100755 index 000000000..4a1b8df0e --- /dev/null +++ b/launch_ci.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +apt update +apt install -y postgresql-client +cd /c2c_ci +python -V +mkdir ~/.venvs +python -m venv ~/.venvs/ci +source ~/.venvs/ci/bin/activate +pip install --upgrade pip setuptools wheel +pip install dotenv flake8 +pip install .[dev] +flake8 c2corg_api es_migration +export PGHOST=postgresql +export PGPORT=5432 +export PGUSER=postgres +export PGPASSWORD=test +./scripts/database/create_test_schema.sh +make CONFIG=config/so.test load-env +curl -v http://elasticsearch:9200 +pytest --cov-report term --cov-report xml --cov=c2corg_api diff --git a/podman-compose.ci.yml b/podman-compose.ci.yml new file mode 100644 index 000000000..011286a0a --- /dev/null +++ b/podman-compose.ci.yml @@ -0,0 +1,26 @@ +services: + postgresql: + image: postgis/postgis + restart: "no" + environment: + POSTGRES_USER: 'postgres' + POSTGRES_PASSWORD: 'test' + ports: + - 5432:5432 + + redis: + image: redis + restart: "no" + ports: + - 6379:6379 + + elasticsearch: + image: elasticsearch:2.4.6-alpine + restart: "no" + + test: + image: python:3.9-bookworm + restart: "no" + volumes: + - ./:/c2c_ci + entrypoint: ["/usr/bin/sleep", "30m"] From 24c84bbfa151393cbe61c1c91869baf0a90cd10d Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Mon, 28 Jul 2025 16:02:03 +0200 Subject: [PATCH 02/41] [CI] Update CI after reverting deploy method in 752b2ea17e --- config/so.test | 25 ++++++++++++------------- launch_ci.sh | 7 ++++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/config/so.test b/config/so.test index 25e56af3e..9057fa2bb 100644 --- a/config/so.test +++ b/config/so.test @@ -1,14 +1,13 @@ -#!/bin/sh +include config/dev -instanceid="github" -base_url="/${instanceid}" -db_name="c2corg_tests" -tests_db_host=postgresql -tests_db_name="c2corg_tests" -elasticsearch_port=9200 -elasticsearch_index="c2corg_tests" -tests_elasticsearch_host=elasticsearch -tests_elasticsearch_port=9200 -tests_elasticsearch_index="c2corg_tests" -redis_url="redis://redis:6379/" -version=0.0.0dev0 +export instanceid = github +export base_url = /${instanceid} +export db_name = c2corg_github_tests +export tests_db_name = c2corg_github_tests +export tests_db_host = postgresql +export elasticsearch_port = 9200 +export elasticsearch_index = c2corg_github_tests +export tests_elasticsearch_host = elasticsearch +export tests_elasticsearch_port = 9200 +export tests_elasticsearch_index = c2corg_github_tests +export redis_url = redis://redis:6379/ diff --git a/launch_ci.sh b/launch_ci.sh index 4a1b8df0e..247f0ef10 100755 --- a/launch_ci.sh +++ b/launch_ci.sh @@ -9,13 +9,14 @@ python -m venv ~/.venvs/ci source ~/.venvs/ci/bin/activate pip install --upgrade pip setuptools wheel pip install dotenv flake8 -pip install .[dev] +pip install -r dev-requirements.txt -r requirements.txt flake8 c2corg_api es_migration export PGHOST=postgresql export PGPORT=5432 export PGUSER=postgres export PGPASSWORD=test -./scripts/database/create_test_schema.sh -make CONFIG=config/so.test load-env +echo "create user \"www-data\" with password 'www-data'" | psql +USER=github scripts/create_user_db_test.sh +make -f config/so.test template curl -v http://elasticsearch:9200 pytest --cov-report term --cov-report xml --cov=c2corg_api From 0dd9ad62377b07c886b57b897f48c7f833ddfc3a Mon Sep 17 00:00:00 2001 From: "floriane.jandot" Date: Thu, 31 Jul 2025 17:22:30 +0200 Subject: [PATCH 03/41] [hotfix] add distincts scripts and change sqlalchemy events --- c2corg_api/__init__.py | 226 ++++++++-------- ...blic_transports_from_France_distinct.bm.sh | 227 ++++++++++++++++ get_public_transports_from_France_distinct.sh | 248 ++++++++++++++++++ get_public_transports_from_Isere_distinct.sh | 248 ++++++++++++++++++ 4 files changed, 845 insertions(+), 104 deletions(-) create mode 100644 get_public_transports_from_France_distinct.bm.sh create mode 100644 get_public_transports_from_France_distinct.sh create mode 100644 get_public_transports_from_Isere_distinct.sh diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 071a6a705..8d94c231e 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -8,7 +8,6 @@ from sqlalchemy import engine_from_config, exc, event from sqlalchemy.pool import Pool from sqlalchemy import text -from dotenv import load_dotenv from c2corg_api.models.document import DocumentGeometry from c2corg_api.models.route import Route @@ -19,7 +18,6 @@ from pyramid.settings import asbool log = logging.getLogger(__name__) -load_dotenv() class RootFactory(ACLDefault): @@ -110,13 +108,16 @@ def process_new_waypoint(mapper, connection, geometry): waypoint_id = geometry.document_id max_distance_waypoint_to_stoparea = int( - os.environ.get("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") ) - walking_speed = float(os.environ.get("WALKING_SPEED")) - max_stop_area = int(os.environ.get("MAX_STOP_AREA_FOR_1_WAYPOINT")) - api_key = os.environ.get("NAVITIA_API_KEY") + walking_speed = float(os.getenv("WALKING_SPEED")) + max_stop_area_for_1_waypoint = int(os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT")) # noqa: E501 + api_key = os.getenv("NAVITIA_API_KEY") max_duration = int(max_distance_waypoint_to_stoparea / walking_speed) + # Augmenter le nombre d'arrêts récupérés pour avoir plus de choix (comme dans le bash) # noqa: E501 + max_stop_area_fetched = max_stop_area_for_1_waypoint * 3 + # Check if document is a waypoint document_type = connection.execute( text( @@ -164,11 +165,11 @@ def process_new_waypoint(mapper, connection, geometry): lon, lat = lon_lat.strip().split(",") - # Navitia request + # Navitia request - récupérer plus d'arrêts pour filtrage places_url = f"https://api.navitia.io/v1/coord/{lon};{lat}/places_nearby" places_params = { "type[]": "stop_area", - "count": max_stop_area, + "count": max_stop_area_fetched, # Plus d'arrêts récupérés "distance": max_distance_waypoint_to_stoparea, } navitia_headers = {"Authorization": api_key} @@ -182,22 +183,61 @@ def process_new_waypoint(mapper, connection, geometry): log.warning(f"No Navitia stops found for the waypoint {waypoint_id}") return - # For each result + # --- NOUVEAU : Filtrage par diversité de transport (comme dans bash) --- + selected_stops = [] + known_transports = set() + selected_count = 0 + + # Traiter les arrêts dans l'ordre (déjà triés par distance par Navitia) for place in places_data["places_nearby"]: if place.get("embedded_type") != "stop_area": continue + if selected_count >= max_stop_area_for_1_waypoint: + break + + stop_id = place["id"] + + # Récupérer les informations de l'arrêt pour connaître ses transports + stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}" + stop_info_response = requests.get(stop_info_url, headers=navitia_headers) # noqa: E501 + stop_info = stop_info_response.json() + + if "places" not in stop_info or not stop_info["places"]: + continue + + # Extraire les transports de cet arrêt + current_stop_transports = set() + for line in stop_info["places"][0]["stop_area"].get("lines", []): + mode = line.get("commercial_mode", {}).get("name", "") + code = line.get("code", "") + transport_key = f"{mode} {code}" + current_stop_transports.add(transport_key) + + # Vérifier si cet arrêt apporte de nouveaux transports + new_transport_found = bool(current_stop_transports - known_transports) + + # Si l'arrêt apporte au moins un nouveau transport, le sélectionner + if new_transport_found: + selected_stops.append(place) + known_transports.update(current_stop_transports) + selected_count += 1 + + log.warning(f"Selected {selected_count} stops out of {len(places_data['places_nearby'])} for waypoint {waypoint_id}") # noqa: E501 + + # Traiter uniquement les arrêts sélectionnés + for place in selected_stops: stop_id = place["id"] stop_name = place["name"] lat_stop = place["stop_area"]["coord"]["lat"] lon_stop = place["stop_area"]["coord"]["lon"] - # Get the travel time by walking + # Get the travel time by walking - utiliser les mêmes paramètres que le bash # noqa: E501 journey_url = "https://api.navitia.io/v1/journeys" journey_params = { "to": f"{lon};{lat}", "walking_speed": walking_speed, - "max_walking_duration_to_pt": max_duration, + "max_walking_direct_path_duration": max_duration, # Paramètre corrigé # noqa: E501 "direct_path_mode[]": "walking", "from": stop_id, "direct_path": "only_with_alternatives", @@ -218,7 +258,7 @@ def process_new_waypoint(mapper, connection, geometry): duration = journey_data["journeys"][0].get("duration", 0) # Convert to distance - distance_km = (duration * walking_speed) / 1000 + distance_km = round((duration * walking_speed) / 1000, 2) # Arrondi à 2 décimales comme bash # noqa: E501 # Check if stop already exists existing_stop_query = text( @@ -232,7 +272,7 @@ def process_new_waypoint(mapper, connection, geometry): ).scalar() if not existing_stop_id: - # Get stop informations + # Get stop informations (déjà récupérées plus haut pour le filtrage) # noqa: E501 stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}" stop_info_response = requests.get( stop_info_url, headers=navitia_headers @@ -242,6 +282,7 @@ def process_new_waypoint(mapper, connection, geometry): if "places" not in stop_info or not stop_info["places"]: continue + # Traiter chaque ligne comme dans le bash for line in stop_info["places"][0]["stop_area"].get("lines", []): line_full_name = line.get("name", "") line_name = line.get("code", "") @@ -253,14 +294,16 @@ def process_new_waypoint(mapper, connection, geometry): """ WITH new_stoparea AS ( INSERT INTO guidebook.stopareas - (navitia_id, stoparea_name, line, operator, geom) VALUES (:stop_id, :stop_name, :line, :operator, ST_Transform(ST_SetSRID(ST_MakePoint(:lon_stop, :lat_stop), 4326), 3857)) + (navitia_id, stoparea_name, line, operator, geom) + VALUES (:stop_id, :stop_name, :line, :operator, + ST_Transform(ST_SetSRID(ST_MakePoint(:lon_stop, :lat_stop), 4326), 3857)) RETURNING stoparea_id ) INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) SELECT stoparea_id, :waypoint_id, :distance_km FROM new_stoparea - """ # noqa: E501 + """ # noqa: E501, W291 ) connection.execute( @@ -316,14 +359,7 @@ def calculate_route_duration(mapper, connection, route): activities = route.activities if route.activities is not None else [] height_diff_up, height_diff_down = _normalize_height_differences(route) - # Vérification du cas spécial pour la grimpe avec seulement - # difficulties_height - if _should_calculate_with_difficulties_only(route, activities): - calculated_duration = _calculate_duration_from_difficulties_only(route, route_id) # noqa: E501 - _update_route_duration(connection, route_id, calculated_duration) - return - - # Calcul standard pour toutes les activités + # Calcul pour toutes les activités et prendre le minimum min_duration = _calculate_min_duration_for_activities( route, activities, height_diff_up, height_diff_down, route_id ) @@ -338,10 +374,11 @@ def _normalize_height_differences(route): height_diff_up = route.height_diff_up height_diff_down = route.height_diff_down - if height_diff_up is None and height_diff_down is not None: - height_diff_up = height_diff_down - elif height_diff_down is None and height_diff_up is not None: + # Règle: si dénivelé négatif absent, égaler au positif + if height_diff_down is None and height_diff_up is not None: height_diff_down = height_diff_up + elif height_diff_up is None and height_diff_down is not None: + height_diff_up = height_diff_down return height_diff_up, height_diff_down @@ -359,55 +396,16 @@ def _get_climbing_activities(): ] -def _should_calculate_with_difficulties_only(route, activities): - """Vérifie s'il faut calculer uniquement avec - height_diff_difficulties.""" - climbing_activities = _get_climbing_activities() - is_climbing = any(activity in climbing_activities for activity in activities) # noqa: E501 - - return ( - is_climbing - and getattr(route, "height_diff_difficulties", None) is not None - and (route.route_length is None or route.height_diff_up is None) - and route.height_diff_difficulties > 0 - ) - - -def _calculate_duration_from_difficulties_only(route, route_id): - """Calcule la durée uniquement basée sur height_diff_difficulties.""" - v_diff = 50.0 # Vitesse ascensionnelle pour les difficultés (m/h) - dm = float(route.height_diff_difficulties) / v_diff - - min_duration_hours = 0.5 - max_duration_hours = 18.0 - - if dm < min_duration_hours or dm > max_duration_hours: - log.warn( - f"Route {route_id}: Calculated duration ({dm:.2f} hours) is out of bounds - setting to NULL" # noqa: E501 - ) - return None - - calculated_duration_in_days = dm / 24.0 - log.warn( - f"Route {route_id}: Database updated with calculated_duration = {calculated_duration_in_days} days (based on difficulties_height only)." # noqa: E501 - ) - return calculated_duration_in_days - - def _calculate_min_duration_for_activities(route, activities, height_diff_up, height_diff_down, route_id): # noqa: E501 """Calcule la durée minimale parmi toutes les activités.""" - h = float(route.route_length if route.route_length is not None else 0) / 1000 # km # noqa: E501 - dp = float(height_diff_up if height_diff_up is not None else 0) # m - dn = float(height_diff_down if height_diff_down is not None else 0) # m - min_duration = None climbing_activities = _get_climbing_activities() for activity in activities: if activity in climbing_activities: - dm = _calculate_climbing_duration(route, h, dp, dn, route_id, activity) # noqa: E501 + dm = _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_id, activity) # noqa: E501 else: - dm = _calculate_standard_duration(activity, h, dp, dn, route_id) + dm = _calculate_standard_duration(activity, route, height_diff_up, height_diff_down, route_id) # noqa: E501 if dm is not None and (min_duration is None or dm < min_duration): min_duration = dm @@ -415,52 +413,70 @@ def _calculate_min_duration_for_activities(route, activities, height_diff_up, he return min_duration -def _calculate_climbing_duration(route, h, dp, dn, route_id, activity): - """Calcule la durée pour les activités de grimpe.""" +def _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_id, activity): # noqa: E501 + """Calcule la durée pour les activités de grimpe selon la logique du bash.""" # noqa: E501 v_diff = 50.0 # Vitesse ascensionnelle pour les difficultés (m/h) - diff_height = getattr(route, "height_diff_difficulties", None) - if diff_height is not None and diff_height > 0: - d_diff = float(diff_height) + h = float(route.route_length if route.route_length is not None else 0) / 1000 # km # noqa: E501 + dp = float(height_diff_up if height_diff_up is not None else 0) # m + dn = float(height_diff_down if height_diff_down is not None else 0) # m - # Vérification de cohérence - if dp > 0 and d_diff > dp: - log.warn( - f"Route {route_id}: Inconsistent difficulties_height ({d_diff}m) > height_diff_up ({dp}m). Skipping for this activity." # noqa: E501 - ) - return None + difficulties_height = getattr(route, "height_diff_difficulties", None) - # Calcul des temps - d_app = max(dp - d_diff, 0) - t_diff = d_diff / v_diff - t_app = _calculate_approach_time(h, d_app, dn) if d_app > 0 else 0 + # CAS 1: Le dénivelé des difficultés n'est pas renseigné + if difficulties_height is None or difficulties_height <= 0: + # On considère que tout l'itinéraire est grimpant et sans approche + if dp <= 0: + return None # Pas de données utilisables pour le calcul - # Temps total selon la formule spéciale - dm = max(t_diff, t_app) + 0.5 * min(t_diff, t_app) - else: - # Si dénivelé des difficultés non disponible, utiliser le - # dénivelé total dm = dp / v_diff + log.warn(f"Calculated climbing route duration for route {route_id} (activity {activity}, no difficulties_height): {dm:.2f} hours") # noqa: E501 + return dm - log.warn( - f"Calculated climbing route duration for route {route_id} (activity {activity}): {dm:.2f} hours" # noqa: E501 - ) + # CAS 2: Le dénivelé des difficultés est renseigné + d_diff = float(difficulties_height) + + # Vérification de cohérence + if dp > 0 and d_diff > dp: + log.warn(f"Route {route_id}: Inconsistent difficulties_height ({d_diff}m) > height_diff_up ({dp}m). Returning NULL.") # noqa: E501 + return None + + # Calcul du temps des difficultés + t_diff = d_diff / v_diff + + # Calcul du dénivelé de l'approche + d_app = max(dp - d_diff, 0) + + # Calcul du temps d'approche + if d_app > 0: + t_app = _calculate_approach_time(h, d_app, dn) + else: + t_app = 0 + + # Calcul final selon le cadrage: max(t_diff, t_app) + 0.5 * min(t_diff, t_app) # noqa: E501 + dm = max(t_diff, t_app) + 0.5 * min(t_diff, t_app) + + log.warn(f"Calculated climbing route duration for route {route_id} (activity {activity}): {dm:.2f} hours (t_diff={t_diff:.2f}, t_app={t_app:.2f})") # noqa: E501 return dm def _calculate_approach_time(h, d_app, dn): - """Calcule le temps d'approche pour la grimpe.""" - v = 5.0 # km/h (vitesse horizontale) + """Calcule le temps d'approche pour la grimpe selon la formule DIN 33466.""" # noqa: E501 + # Paramètres pour l'approche (randonnée) + v = 5.0 # km/h (vitesse horizontale) a = 300.0 # m/h (montée) d = 500.0 # m/h (descente) - dh = h / v - dv = (d_app / a) + (dn / d) + dh_app = h / v # Composante horizontale de l'approche + dv_app = (d_app / a) + (dn / d) # Composante verticale de l'approche (montée + descente) # noqa: E501 - if dh < dv: - return dv + (dh / 2) + # Appliquer la formule DIN 33466 pour le temps d'approche + if dh_app < dv_app: + t_app = dv_app + (dh_app / 2) else: - return (dv / 2) + dh + t_app = (dv_app / 2) + dh_app + + return t_app def _get_activity_parameters(activity): @@ -474,22 +490,24 @@ def _get_activity_parameters(activity): return parameters.get(activity, (5.0, 300.0, 500.0)) # Valeurs par défaut -def _calculate_standard_duration(activity, h, dp, dn, route_id): - """Calcule la durée pour les activités standard (non grimpantes).""" +def _calculate_standard_duration(activity, route, height_diff_up, height_diff_down, route_id): # noqa: E501 + """Calcule la durée pour les activités standard (non grimpantes) selon DIN 33466.""" # noqa: E501 v, a, d = _get_activity_parameters(activity) + h = float(route.route_length if route.route_length is not None else 0) / 1000 # km # noqa: E501 + dp = float(height_diff_up if height_diff_up is not None else 0) # m + dn = float(height_diff_down if height_diff_down is not None else 0) # m + dh = h / v # durée basée sur la distance horizontale dv = (dp / a) + (dn / d) # durée basée sur les dénivelés - # Calcul de la durée finale en heures + # Calcul de la durée finale en heures selon DIN 33466 if dh < dv: dm = dv + (dh / 2) else: dm = (dv / 2) + dh - log.warn( - f"Calculated standard route duration for route {route_id} (activity {activity}): {dm:.2f} hours" # noqa: E501 - ) + log.warn(f"Calculated standard route duration for route {route_id} (activity {activity}): {dm:.2f} hours") # noqa: E501 return dm @@ -504,7 +522,7 @@ def _validate_and_convert_duration(min_duration, route_id): or min_duration > max_duration_hours ): log.warn( - f"Route {route_id}: Calculated duration ({min_duration:.2f} hours) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 + f"Route {route_id}: Calculated duration ({min_duration:.2f} hours if not None) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 ) return None diff --git a/get_public_transports_from_France_distinct.bm.sh b/get_public_transports_from_France_distinct.bm.sh new file mode 100644 index 000000000..8a81e788d --- /dev/null +++ b/get_public_transports_from_France_distinct.bm.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# shellcheck disable=SC2001 + +DURATION=$(echo "scale=0; $MAX_DISTANCE_WAYPOINT_TO_STOPAREA / $WALKING_SPEED" | bc) + +API_PORT=${API_PORT:-6543} + +BASE_API_URL="http://localhost:${API_PORT}/waypoints?wtyp=access&a=14274&limit=100" +OUTPUT_FILE="/tmp/waypoints_ids.txt" +LOG_FILE="log-navitia.txt" +NAVITIA_REQUEST_COUNT=0 +SQL_FILE="/tmp/sql_commands.sql" + +# Augmenter le nombre d'arrêts récupérés pour avoir plus de choix +MAX_STOP_AREA_FETCHED=$((MAX_STOP_AREA_FOR_1_WAYPOINT * 3)) # Récupérer 6 fois plus d'arrêts + +echo "Start time :" > "$LOG_FILE" +echo $(date +"%Y-%m-%d-%H-%M-%S") >> "$LOG_FILE" + +# --- Pagination logic --- +OFFSET=0 +LIMIT=100 # The default API limit +TOTAL_WAYPOINTS=0 + + +> "$OUTPUT_FILE" + +while true; do + API_URL="${BASE_API_URL}&offset=${OFFSET}" + echo "Fetching: $API_URL" >> "$LOG_FILE" + WAYPOINTS_IDS=$(curl -s "$API_URL" | jq -r '.documents[] | .document_id' 2>/dev/null) + + nb_current_waypoints=$(echo "$WAYPOINTS_IDS" | wc -l) + + # If jq returned one line (no waypoint message), or if wc -l returned 0 ==> there is nothing left + if [ "$nb_current_waypoints" -eq 1 ]; then + echo "No more waypoints to fetch. Breaking loop." >> "$LOG_FILE" + break + fi + + echo "$WAYPOINTS_IDS" >> "$OUTPUT_FILE" + + TOTAL_WAYPOINTS=$((TOTAL_WAYPOINTS + nb_current_waypoints)) + OFFSET=$((OFFSET + LIMIT)) + +done + +nb_waypoints=$(wc -l < "$OUTPUT_FILE") +echo "Total waypoints fetched: $nb_waypoints" >> "$LOG_FILE" +echo "Total waypoints fetched: $nb_waypoints" + + +# Initialize SQL file +> "$SQL_FILE" + +psql -t -c "TRUNCATE TABLE guidebook.waypoints_stopareas RESTART IDENTITY;" +psql -t -c "TRUNCATE TABLE guidebook.stopareas RESTART IDENTITY;" + + + + +for ((k=1; k<=nb_waypoints; k++)); do + # Log progress every 10 waypoints + if (( k % 10 == 0 )) || (( k == 1 )); then + echo "Progress: $k/$nb_waypoints waypoints processed. Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + fi + + WAYPOINT_ID=$(sed "${k}q;d" /tmp/waypoints_ids.txt) + + # Get waypoint coordinates from backend + lon_lat=$(psql -t -c " + SELECT ST_X(ST_Transform(geom, 4326)) || ',' || ST_Y(ST_Transform(geom, 4326)) + FROM guidebook.documents_geometries + WHERE document_id = $WAYPOINT_ID; + " | tr -d ' ') + + # Extract longitude and latitude + lon=$(echo "$lon_lat" | cut -d',' -f1) + lat=$(echo "$lon_lat" | cut -d',' -f2) + + # Check that coordinates were retrieved successfully + if [[ -z "$lon" || -z "$lat" || "$lon" == "null" || "$lat" == "null" ]]; then + continue + fi + + # Query Navitia to retrieve nearby stopareas (récupérer plus d'arrêts) + response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/coord/$lon%3B$lat/places_nearby?type%5B%5D=stop_area&count=$MAX_STOP_AREA_FETCHED&distance=$MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + ((NAVITIA_REQUEST_COUNT++)) + + has_places=$(echo "$response" | jq 'has("places_nearby") and (.places_nearby | length > 0)') + + if [[ "$has_places" == "true" ]]; then + # Extract all stop names and IDs to temporary files + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .name' > /tmp/stop_names.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .id' > /tmp/stop_ids.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .stop_area.coord.lat' > /tmp/lat.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .stop_area.coord.lon' > /tmp/lon.txt + + # Count the number of stops + stop_area_count=$(wc -l < /tmp/stop_ids.txt) + + # --- NOUVEAU : Filtrage par diversité de transport --- + > /tmp/selected_stops.txt + > /tmp/known_transports.txt + selected_count=0 + + # Traiter les arrêts dans l'ordre (déjà triés par distance par Navitia) + for ((i=1; i<=stop_area_count && selected_count /tmp/current_stop_transports.txt + + # Vérifier si cet arrêt apporte de nouveaux transports + new_transport_found=false + current_transport_count=$(wc -l < /tmp/current_stop_transports.txt) + + for ((t=1; t<=current_transport_count; t++)); do + transport=$(sed "${t}q;d" /tmp/current_stop_transports.txt) + if ! grep -Fxq "$transport" /tmp/known_transports.txt; then + new_transport_found=true + echo "$transport" >> /tmp/known_transports.txt + fi + done + + # Si l'arrêt apporte au moins un nouveau transport, le sélectionner + if [ "$new_transport_found" = true ]; then + echo "$i" >> /tmp/selected_stops.txt + ((selected_count++)) + fi + done + + echo "Selected $selected_count stops out of $stop_area_count for waypoint $WAYPOINT_ID" >> "$LOG_FILE" + + # Traiter uniquement les arrêts sélectionnés + selected_stops_count=$(wc -l < /tmp/selected_stops.txt) + for ((s=1; s<=selected_stops_count; s++)); do + stop_index=$(sed "${s}q;d" /tmp/selected_stops.txt) + stop_name=$(sed "${stop_index}q;d" /tmp/stop_names.txt) + stop_id=$(sed "${stop_index}q;d" /tmp/stop_ids.txt) + lat_stop=$(sed "${stop_index}q;d" /tmp/lat.txt) + lon_stop=$(sed "${stop_index}q;d" /tmp/lon.txt) + + # Get walking travel time via Navitia + journey_response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/journeys?to=$lon%3B$lat&walking_speed=$WALKING_SPEED&max_walking_direct_path_duration=$DURATION&direct_path_mode%5B%5D=walking&from=$stop_id&direct_path=only_with_alternatives") + ((NAVITIA_REQUEST_COUNT++)) + + # Check if Navitia found a solution or returns an error + has_error=$(echo "$journey_response" | jq -r 'has("error")') + + if [[ "$has_error" == "true" ]]; then + continue + fi + + # Extract walking travel time (in seconds) + duration=$(echo "$journey_response" | jq -r '.journeys[0].duration // 0') + + # Convert duration to distance (1.12m/s => 1.12 * duration / 1000 to get km) + distance_km=$(awk "BEGIN {printf \"%.2f\", ($duration * $WALKING_SPEED) / 1000}") + + # Check if the stop already exists + existing_stop_id=$(psql -t -c "SELECT stoparea_id FROM guidebook.stopareas WHERE navitia_id = '$stop_id' LIMIT 1;" | tr -d ' \n\r') + + # For new stop areas + if [[ -z "$existing_stop_id" ]]; then + # Get the stop_area information + stop_info=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/places/$stop_id") + ((NAVITIA_REQUEST_COUNT++)) + + # Loop through lines + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].name' > /tmp/lines.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].code' > /tmp/code.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].network.name' > /tmp/network.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].commercial_mode.name' > /tmp/mode.txt + + # Count the number of lines + stop_count=$(wc -l < /tmp/lines.txt) + + # Process each line + for ((j=1; j<=stop_count; j++)); do + line_full_name=$(sed "${j}q;d" /tmp/lines.txt) + line_name=$(sed "${j}q;d" /tmp/code.txt) + operator_name=$(sed "${j}q;d" /tmp/network.txt) + mode=$(sed "${j}q;d" /tmp/mode.txt) + + # Create a stoparea document and save its ID + # shellcheck disable=SC2001 + echo "DO \$\$ + DECLARE stoparea_doc_id integer; + BEGIN + -- Insert stopareas + INSERT INTO guidebook.stopareas (navitia_id, stoparea_name, line, operator, geom) + VALUES ('$stop_id', '$(echo "$stop_name" | sed "s/'/''/g")', '$mode $line_name - $(echo "$line_full_name" | sed "s/'/''/g")', '$(echo "$operator_name" | sed "s/'/''/g")', ST_Transform(ST_SetSRID(ST_MakePoint($lon_stop, $lat_stop), 4326), 3857)) + RETURNING stoparea_id INTO stoparea_doc_id; + + -- Insert relationship + INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) + VALUES (stoparea_doc_id, $WAYPOINT_ID, $distance_km); + END \$\$;" >> "$SQL_FILE" + done + rm /tmp/lines.txt /tmp/code.txt /tmp/network.txt /tmp/mode.txt + else + # For existing stop areas + echo "INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) VALUES ($existing_stop_id, $WAYPOINT_ID, $distance_km);" >> "$SQL_FILE" + fi + done + + # Cleanup + rm /tmp/stop_names.txt /tmp/stop_ids.txt /tmp/lat.txt /tmp/lon.txt /tmp/selected_stops.txt /tmp/known_transports.txt + fi +done + +# Log final progress +echo "Completed: $nb_waypoints/$nb_waypoints waypoints processed. Total Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + +echo "Stop time :" >> $LOG_FILE +echo $(date +"%Y-%m-%d-%H-%M-%S") >> $LOG_FILE + +# Execute all SQL commands in one go +echo "Sql file length : $(wc -l < "$SQL_FILE") lines." >> $LOG_FILE +psql -q < /tmp/sql_commands.sql + +echo "Inserts done." >> $LOG_FILE diff --git a/get_public_transports_from_France_distinct.sh b/get_public_transports_from_France_distinct.sh new file mode 100644 index 000000000..334c36a0e --- /dev/null +++ b/get_public_transports_from_France_distinct.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# shellcheck disable=SC2001 + +# Configuration +SERVICE_NAME="postgresql" +DB_USER="postgres" +DB_NAME="c2corg" + +if [ -f ./.env ]; then + # Load .env data + export $(grep -v '^#' ./.env | xargs) +else + echo ".env file not found!" + exit 1 +fi + +DURATION=$(echo "scale=0; $MAX_DISTANCE_WAYPOINT_TO_STOPAREA / $WALKING_SPEED" | bc) + +PROJECT_NAME=${PROJECT_NAME:-""} +API_PORT=${API_PORT:-6543} +CCOMPOSE=${CCOMPOSE:-"podman-compose"} +STANDALONE=${PODMAN_ENV:-""} + +BASE_API_URL="http://localhost:${API_PORT}/waypoints?wtyp=access&a=14274&limit=100" +OUTPUT_FILE="/tmp/waypoints_ids.txt" +LOG_FILE="log-navitia.txt" +NAVITIA_REQUEST_COUNT=0 +SQL_FILE="/tmp/sql_commands.sql" + +# Augmenter le nombre d'arrêts récupérés pour avoir plus de choix +MAX_STOP_AREA_FETCHED=$((MAX_STOP_AREA_FOR_1_WAYPOINT * 3)) + +if [[ -n "$STANDALONE" ]]; then + SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + cd "$SCRIPTPATH"/.. || exit +fi + +echo "Start time :" > "$LOG_FILE" +echo $(date +"%Y-%m-%d-%H-%M-%S") >> "$LOG_FILE" + +# --- Pagination logic --- +OFFSET=0 +LIMIT=100 # The default API limit +TOTAL_WAYPOINTS=0 + + +> "$OUTPUT_FILE" + +while true; do + API_URL="${BASE_API_URL}&offset=${OFFSET}" + echo "Fetching: $API_URL" >> "$LOG_FILE" + WAYPOINTS_IDS=$(curl -s "$API_URL" | jq -r '.documents[] | .document_id' 2>/dev/null) + + nb_current_waypoints=$(echo "$WAYPOINTS_IDS" | wc -l) + + # If jq returned one line (no waypoint message), or if wc -l returned 0 ==> there is nothing left + if [ "$nb_current_waypoints" -eq 1 ]; then + echo "No more waypoints to fetch. Breaking loop." >> "$LOG_FILE" + break + fi + + echo "$WAYPOINTS_IDS" >> "$OUTPUT_FILE" + + TOTAL_WAYPOINTS=$((TOTAL_WAYPOINTS + nb_current_waypoints)) + OFFSET=$((OFFSET + LIMIT)) + +done + +nb_waypoints=$(wc -l < "$OUTPUT_FILE") +echo "Total waypoints fetched: $nb_waypoints" >> "$LOG_FILE" +echo "Total waypoints fetched: $nb_waypoints" + + +# Initialize SQL file +> "$SQL_FILE" + +$CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c "TRUNCATE TABLE guidebook.waypoints_stopareas RESTART IDENTITY;" +$CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c "TRUNCATE TABLE guidebook.stopareas RESTART IDENTITY;" + + + + +for ((k=1; k<=nb_waypoints; k++)); do + # Log progress every 10 waypoints + if (( k % 10 == 0 )) || (( k == 1 )); then + echo "Progress: $k/$nb_waypoints waypoints processed. Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + fi + + WAYPOINT_ID=$(sed "${k}q;d" /tmp/waypoints_ids.txt) + + # Get waypoint coordinates from backend + lon_lat=$($CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c " + SELECT ST_X(ST_Transform(geom, 4326)) || ',' || ST_Y(ST_Transform(geom, 4326)) + FROM guidebook.documents_geometries + WHERE document_id = $WAYPOINT_ID; + " | tr -d ' ') + + # Extract longitude and latitude + lon=$(echo "$lon_lat" | cut -d',' -f1) + lat=$(echo "$lon_lat" | cut -d',' -f2) + + # Check that coordinates were retrieved successfully + if [[ -z "$lon" || -z "$lat" || "$lon" == "null" || "$lat" == "null" ]]; then + continue + fi + + # Query Navitia to retrieve nearby stopareas (récupérer plus d'arrêts) + response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/coord/$lon%3B$lat/places_nearby?type%5B%5D=stop_area&count=$MAX_STOP_AREA_FETCHED&distance=$MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + ((NAVITIA_REQUEST_COUNT++)) + + has_places=$(echo "$response" | jq 'has("places_nearby") and (.places_nearby | length > 0)') + + if [[ "$has_places" == "true" ]]; then + # Extract all stop names and IDs to temporary files + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .name' > /tmp/stop_names.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .id' > /tmp/stop_ids.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .stop_area.coord.lat' > /tmp/lat.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .stop_area.coord.lon' > /tmp/lon.txt + + # Count the number of stops + stop_area_count=$(wc -l < /tmp/stop_ids.txt) + + # --- NOUVEAU : Filtrage par diversité de transport --- + > /tmp/selected_stops.txt + > /tmp/known_transports.txt + selected_count=0 + + # Traiter les arrêts dans l'ordre (déjà triés par distance par Navitia) + for ((i=1; i<=stop_area_count && selected_count /tmp/current_stop_transports.txt + + # Vérifier si cet arrêt apporte de nouveaux transports + new_transport_found=false + current_transport_count=$(wc -l < /tmp/current_stop_transports.txt) + + for ((t=1; t<=current_transport_count; t++)); do + transport=$(sed "${t}q;d" /tmp/current_stop_transports.txt) + if ! grep -Fxq "$transport" /tmp/known_transports.txt; then + new_transport_found=true + echo "$transport" >> /tmp/known_transports.txt + fi + done + + # Si l'arrêt apporte au moins un nouveau transport, le sélectionner + if [ "$new_transport_found" = true ]; then + echo "$i" >> /tmp/selected_stops.txt + ((selected_count++)) + fi + done + + echo "Selected $selected_count stops out of $stop_area_count for waypoint $WAYPOINT_ID" >> "$LOG_FILE" + + # Traiter uniquement les arrêts sélectionnés + selected_stops_count=$(wc -l < /tmp/selected_stops.txt) + for ((s=1; s<=selected_stops_count; s++)); do + stop_index=$(sed "${s}q;d" /tmp/selected_stops.txt) + stop_name=$(sed "${stop_index}q;d" /tmp/stop_names.txt) + stop_id=$(sed "${stop_index}q;d" /tmp/stop_ids.txt) + lat_stop=$(sed "${stop_index}q;d" /tmp/lat.txt) + lon_stop=$(sed "${stop_index}q;d" /tmp/lon.txt) + + # Get walking travel time via Navitia + journey_response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/journeys?to=$lon%3B$lat&walking_speed=$WALKING_SPEED&max_walking_direct_path_duration=$DURATION&direct_path_mode%5B%5D=walking&from=$stop_id&direct_path=only_with_alternatives") + ((NAVITIA_REQUEST_COUNT++)) + + # Check if Navitia found a solution or returns an error + has_error=$(echo "$journey_response" | jq -r 'has("error")') + + if [[ "$has_error" == "true" ]]; then + continue + fi + + # Extract walking travel time (in seconds) + duration=$(echo "$journey_response" | jq -r '.journeys[0].duration // 0') + + # Convert duration to distance (1.12m/s => 1.12 * duration / 1000 to get km) + distance_km=$(awk "BEGIN {printf \"%.2f\", ($duration * $WALKING_SPEED) / 1000}") + + # Check if the stop already exists + existing_stop_id=$($CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c "SELECT stoparea_id FROM guidebook.stopareas WHERE navitia_id = '$stop_id' LIMIT 1;" | tr -d ' \n\r') + + # For new stop areas + if [[ -z "$existing_stop_id" ]]; then + # Get the stop_area information + stop_info=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/places/$stop_id") + ((NAVITIA_REQUEST_COUNT++)) + + # Loop through lines + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].name' > /tmp/lines.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].code' > /tmp/code.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].network.name' > /tmp/network.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].commercial_mode.name' > /tmp/mode.txt + + # Count the number of lines + stop_count=$(wc -l < /tmp/lines.txt) + + # Process each line + for ((j=1; j<=stop_count; j++)); do + line_full_name=$(sed "${j}q;d" /tmp/lines.txt) + line_name=$(sed "${j}q;d" /tmp/code.txt) + operator_name=$(sed "${j}q;d" /tmp/network.txt) + mode=$(sed "${j}q;d" /tmp/mode.txt) + + # Create a stoparea document and save its ID + # shellcheck disable=SC2001 + echo "DO \$\$ + DECLARE stoparea_doc_id integer; + BEGIN + -- Insert stopareas + INSERT INTO guidebook.stopareas (navitia_id, stoparea_name, line, operator, geom) + VALUES ('$stop_id', '$(echo "$stop_name" | sed "s/'/''/g")', '$mode $line_name - $(echo "$line_full_name" | sed "s/'/''/g")', '$(echo "$operator_name" | sed "s/'/''/g")', ST_Transform(ST_SetSRID(ST_MakePoint($lon_stop, $lat_stop), 4326), 3857)) + RETURNING stoparea_id INTO stoparea_doc_id; + + -- Insert relationship + INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) + VALUES (stoparea_doc_id, $WAYPOINT_ID, $distance_km); + END \$\$;" >> "$SQL_FILE" + done + rm /tmp/lines.txt /tmp/code.txt /tmp/network.txt /tmp/mode.txt + else + # For existing stop areas + echo "INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) VALUES ($existing_stop_id, $WAYPOINT_ID, $distance_km);" >> "$SQL_FILE" + fi + done + + # Cleanup + rm /tmp/stop_names.txt /tmp/stop_ids.txt /tmp/lat.txt /tmp/lon.txt /tmp/selected_stops.txt /tmp/known_transports.txt + fi +done + +# Log final progress +echo "Completed: $nb_waypoints/$nb_waypoints waypoints processed. Total Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + +echo "Stop time :" >> $LOG_FILE +echo $(date +"%Y-%m-%d-%H-%M-%S") >> $LOG_FILE + +# Execute all SQL commands in one go +echo "Sql file length : $(wc -l < "$SQL_FILE") lines." >> $LOG_FILE +$CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -q -U $DB_USER -d $DB_NAME < /tmp/sql_commands.sql + +echo "Inserts done." >> $LOG_FILE diff --git a/get_public_transports_from_Isere_distinct.sh b/get_public_transports_from_Isere_distinct.sh new file mode 100644 index 000000000..0d7d462f4 --- /dev/null +++ b/get_public_transports_from_Isere_distinct.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# shellcheck disable=SC2001 + +# Configuration +SERVICE_NAME="postgresql" +DB_USER="postgres" +DB_NAME="c2corg" + +if [ -f ./.env ]; then + # Load .env data + export $(grep -v '^#' ./.env | xargs) +else + echo ".env file not found!" + exit 1 +fi + +DURATION=$(echo "scale=0; $MAX_DISTANCE_WAYPOINT_TO_STOPAREA / $WALKING_SPEED" | bc) + +PROJECT_NAME=${PROJECT_NAME:-""} +API_PORT=${API_PORT:-6543} +CCOMPOSE=${CCOMPOSE:-"podman-compose"} +STANDALONE=${PODMAN_ENV:-""} + +BASE_API_URL="http://localhost:${API_PORT}/waypoints?wtyp=access&a=14328&limit=100" +OUTPUT_FILE="/tmp/waypoints_ids.txt" +LOG_FILE="log-navitia.txt" +NAVITIA_REQUEST_COUNT=0 +SQL_FILE="/tmp/sql_commands.sql" + +# Augmenter le nombre d'arrêts récupérés pour avoir plus de choix +MAX_STOP_AREA_FETCHED=$((MAX_STOP_AREA_FOR_1_WAYPOINT * 3)) + +if [[ -n "$STANDALONE" ]]; then + SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + cd "$SCRIPTPATH"/.. || exit +fi + +echo "Start time :" > "$LOG_FILE" +echo $(date +"%Y-%m-%d-%H-%M-%S") >> "$LOG_FILE" + +# --- Pagination logic --- +OFFSET=0 +LIMIT=100 # The default API limit +TOTAL_WAYPOINTS=0 + + +> "$OUTPUT_FILE" + +while true; do + API_URL="${BASE_API_URL}&offset=${OFFSET}" + echo "Fetching: $API_URL" >> "$LOG_FILE" + WAYPOINTS_IDS=$(curl -s "$API_URL" | jq -r '.documents[] | .document_id' 2>/dev/null) + + nb_current_waypoints=$(echo "$WAYPOINTS_IDS" | wc -l) + + # If jq returned one line (no waypoint message), or if wc -l returned 0 ==> there is nothing left + if [ "$nb_current_waypoints" -eq 1 ]; then + echo "No more waypoints to fetch. Breaking loop." >> "$LOG_FILE" + break + fi + + echo "$WAYPOINTS_IDS" >> "$OUTPUT_FILE" + + TOTAL_WAYPOINTS=$((TOTAL_WAYPOINTS + nb_current_waypoints)) + OFFSET=$((OFFSET + LIMIT)) + +done + +nb_waypoints=$(wc -l < "$OUTPUT_FILE") +echo "Total waypoints fetched: $nb_waypoints" >> "$LOG_FILE" +echo "Total waypoints fetched: $nb_waypoints" + + +# Initialize SQL file +> "$SQL_FILE" + +$CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c "TRUNCATE TABLE guidebook.waypoints_stopareas RESTART IDENTITY;" +$CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c "TRUNCATE TABLE guidebook.stopareas RESTART IDENTITY;" + + + + +for ((k=1; k<=nb_waypoints; k++)); do + # Log progress every 10 waypoints + if (( k % 10 == 0 )) || (( k == 1 )); then + echo "Progress: $k/$nb_waypoints waypoints processed. Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + fi + + WAYPOINT_ID=$(sed "${k}q;d" /tmp/waypoints_ids.txt) + + # Get waypoint coordinates from backend + lon_lat=$($CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c " + SELECT ST_X(ST_Transform(geom, 4326)) || ',' || ST_Y(ST_Transform(geom, 4326)) + FROM guidebook.documents_geometries + WHERE document_id = $WAYPOINT_ID; + " | tr -d ' ') + + # Extract longitude and latitude + lon=$(echo "$lon_lat" | cut -d',' -f1) + lat=$(echo "$lon_lat" | cut -d',' -f2) + + # Check that coordinates were retrieved successfully + if [[ -z "$lon" || -z "$lat" || "$lon" == "null" || "$lat" == "null" ]]; then + continue + fi + + # Query Navitia to retrieve nearby stopareas (récupérer plus d'arrêts) + response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/coord/$lon%3B$lat/places_nearby?type%5B%5D=stop_area&count=$MAX_STOP_AREA_FETCHED&distance=$MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + ((NAVITIA_REQUEST_COUNT++)) + + has_places=$(echo "$response" | jq 'has("places_nearby") and (.places_nearby | length > 0)') + + if [[ "$has_places" == "true" ]]; then + # Extract all stop names and IDs to temporary files + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .name' > /tmp/stop_names.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .id' > /tmp/stop_ids.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .stop_area.coord.lat' > /tmp/lat.txt + echo "$response" | jq -r '.places_nearby[] | select(.embedded_type == "stop_area") | .stop_area.coord.lon' > /tmp/lon.txt + + # Count the number of stops + stop_area_count=$(wc -l < /tmp/stop_ids.txt) + + # --- NOUVEAU : Filtrage par diversité de transport --- + > /tmp/selected_stops.txt + > /tmp/known_transports.txt + selected_count=0 + + # Traiter les arrêts dans l'ordre (déjà triés par distance par Navitia) + for ((i=1; i<=stop_area_count && selected_count /tmp/current_stop_transports.txt + + # Vérifier si cet arrêt apporte de nouveaux transports + new_transport_found=false + current_transport_count=$(wc -l < /tmp/current_stop_transports.txt) + + for ((t=1; t<=current_transport_count; t++)); do + transport=$(sed "${t}q;d" /tmp/current_stop_transports.txt) + if ! grep -Fxq "$transport" /tmp/known_transports.txt; then + new_transport_found=true + echo "$transport" >> /tmp/known_transports.txt + fi + done + + # Si l'arrêt apporte au moins un nouveau transport, le sélectionner + if [ "$new_transport_found" = true ]; then + echo "$i" >> /tmp/selected_stops.txt + ((selected_count++)) + fi + done + + echo "Selected $selected_count stops out of $stop_area_count for waypoint $WAYPOINT_ID" >> "$LOG_FILE" + + # Traiter uniquement les arrêts sélectionnés + selected_stops_count=$(wc -l < /tmp/selected_stops.txt) + for ((s=1; s<=selected_stops_count; s++)); do + stop_index=$(sed "${s}q;d" /tmp/selected_stops.txt) + stop_name=$(sed "${stop_index}q;d" /tmp/stop_names.txt) + stop_id=$(sed "${stop_index}q;d" /tmp/stop_ids.txt) + lat_stop=$(sed "${stop_index}q;d" /tmp/lat.txt) + lon_stop=$(sed "${stop_index}q;d" /tmp/lon.txt) + + # Get walking travel time via Navitia + journey_response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/journeys?to=$lon%3B$lat&walking_speed=$WALKING_SPEED&max_walking_direct_path_duration=$DURATION&direct_path_mode%5B%5D=walking&from=$stop_id&direct_path=only_with_alternatives") + ((NAVITIA_REQUEST_COUNT++)) + + # Check if Navitia found a solution or returns an error + has_error=$(echo "$journey_response" | jq -r 'has("error")') + + if [[ "$has_error" == "true" ]]; then + continue + fi + + # Extract walking travel time (in seconds) + duration=$(echo "$journey_response" | jq -r '.journeys[0].duration // 0') + + # Convert duration to distance (1.12m/s => 1.12 * duration / 1000 to get km) + distance_km=$(awk "BEGIN {printf \"%.2f\", ($duration * $WALKING_SPEED) / 1000}") + + # Check if the stop already exists + existing_stop_id=$($CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -U $DB_USER -d $DB_NAME -t -c "SELECT stoparea_id FROM guidebook.stopareas WHERE navitia_id = '$stop_id' LIMIT 1;" | tr -d ' \n\r') + + # For new stop areas + if [[ -z "$existing_stop_id" ]]; then + # Get the stop_area information + stop_info=$(curl -s -H "Authorization: $NAVITIA_API_KEY" "https://api.navitia.io/v1/places/$stop_id") + ((NAVITIA_REQUEST_COUNT++)) + + # Loop through lines + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].name' > /tmp/lines.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].code' > /tmp/code.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].network.name' > /tmp/network.txt + echo "$stop_info" | jq -r '.places[0].stop_area.lines[].commercial_mode.name' > /tmp/mode.txt + + # Count the number of lines + stop_count=$(wc -l < /tmp/lines.txt) + + # Process each line + for ((j=1; j<=stop_count; j++)); do + line_full_name=$(sed "${j}q;d" /tmp/lines.txt) + line_name=$(sed "${j}q;d" /tmp/code.txt) + operator_name=$(sed "${j}q;d" /tmp/network.txt) + mode=$(sed "${j}q;d" /tmp/mode.txt) + + # Create a stoparea document and save its ID + # shellcheck disable=SC2001 + echo "DO \$\$ + DECLARE stoparea_doc_id integer; + BEGIN + -- Insert stopareas + INSERT INTO guidebook.stopareas (navitia_id, stoparea_name, line, operator, geom) + VALUES ('$stop_id', '$(echo "$stop_name" | sed "s/'/''/g")', '$mode $line_name - $(echo "$line_full_name" | sed "s/'/''/g")', '$(echo "$operator_name" | sed "s/'/''/g")', ST_Transform(ST_SetSRID(ST_MakePoint($lon_stop, $lat_stop), 4326), 3857)) + RETURNING stoparea_id INTO stoparea_doc_id; + + -- Insert relationship + INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) + VALUES (stoparea_doc_id, $WAYPOINT_ID, $distance_km); + END \$\$;" >> "$SQL_FILE" + done + rm /tmp/lines.txt /tmp/code.txt /tmp/network.txt /tmp/mode.txt + else + # For existing stop areas + echo "INSERT INTO guidebook.waypoints_stopareas (stoparea_id, waypoint_id, distance) VALUES ($existing_stop_id, $WAYPOINT_ID, $distance_km);" >> "$SQL_FILE" + fi + done + + # Cleanup + rm /tmp/stop_names.txt /tmp/stop_ids.txt /tmp/lat.txt /tmp/lon.txt /tmp/selected_stops.txt /tmp/known_transports.txt + fi +done + +# Log final progress +echo "Completed: $nb_waypoints/$nb_waypoints waypoints processed. Total Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + +echo "Stop time :" >> $LOG_FILE +echo $(date +"%Y-%m-%d-%H-%M-%S") >> $LOG_FILE + +# Execute all SQL commands in one go +echo "Sql file length : $(wc -l < "$SQL_FILE") lines." >> $LOG_FILE +$CCOMPOSE -p "${PROJECT_NAME}" exec -T $SERVICE_NAME psql -q -U $DB_USER -d $DB_NAME < /tmp/sql_commands.sql + +echo "Inserts done." >> $LOG_FILE From 905fa55ce2919021cad24d9bfdc6ff7e01e036af Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Thu, 31 Jul 2025 18:18:21 +0200 Subject: [PATCH 04/41] [CI] source env for CI --- launch_ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/launch_ci.sh b/launch_ci.sh index 247f0ef10..45749310a 100755 --- a/launch_ci.sh +++ b/launch_ci.sh @@ -19,4 +19,5 @@ echo "create user \"www-data\" with password 'www-data'" | psql USER=github scripts/create_user_db_test.sh make -f config/so.test template curl -v http://elasticsearch:9200 +export $(cat .env | grep -v "^#" | xargs) pytest --cov-report term --cov-report xml --cov=c2corg_api From b11143bf5785a8cb2e3337a1614e618f1c00ade0 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Thu, 11 Sep 2025 11:49:06 +0200 Subject: [PATCH 05/41] [fix] support updating TC when waypoint is modified and optimize navitia requests --- c2corg_api/__init__.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 8d94c231e..2d9e2cde8 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -102,9 +102,11 @@ def configure_anonymous(settings, config): @event.listens_for(DocumentGeometry, "after_insert") +@event.listens_for(DocumentGeometry, "after_update") def process_new_waypoint(mapper, connection, geometry): """Processes a new waypoint to find its public transports after inserting it into documents_geometries.""" + log.debug("Entering process_new_waypoint callback") waypoint_id = geometry.document_id max_distance_waypoint_to_stoparea = int( @@ -219,10 +221,26 @@ def process_new_waypoint(mapper, connection, geometry): # Si l'arrêt apporte au moins un nouveau transport, le sélectionner if new_transport_found: + place["stop_info"] = stop_info selected_stops.append(place) known_transports.update(current_stop_transports) selected_count += 1 + # Delete existing stopareas for waypoint + delete_relation_query = text( + """ + DELETE FROM guidebook.waypoints_stopareas + WHERE waypoint_id = :waypoint_id + """ + ) + + connection.execute( + delete_relation_query, + { + "waypoint_id": waypoint_id, + }, + ) + log.warning(f"Selected {selected_count} stops out of {len(places_data['places_nearby'])} for waypoint {waypoint_id}") # noqa: E501 # Traiter uniquement les arrêts sélectionnés @@ -272,15 +290,7 @@ def process_new_waypoint(mapper, connection, geometry): ).scalar() if not existing_stop_id: - # Get stop informations (déjà récupérées plus haut pour le filtrage) # noqa: E501 - stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}" - stop_info_response = requests.get( - stop_info_url, headers=navitia_headers - ) - stop_info = stop_info_response.json() - - if "places" not in stop_info or not stop_info["places"]: - continue + stop_info = place["stop_info"] # Traiter chaque ligne comme dans le bash for line in stop_info["places"][0]["stop_area"].get("lines", []): From 288e183f8bd03f96bccc589201cc102c3b3fba3e Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Thu, 11 Sep 2025 13:29:31 +0200 Subject: [PATCH 06/41] [scripts] make distinct navitia scripts executable --- get_public_transports_from_France_distinct.bm.sh | 0 get_public_transports_from_France_distinct.sh | 0 get_public_transports_from_Isere_distinct.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 get_public_transports_from_France_distinct.bm.sh mode change 100644 => 100755 get_public_transports_from_France_distinct.sh mode change 100644 => 100755 get_public_transports_from_Isere_distinct.sh diff --git a/get_public_transports_from_France_distinct.bm.sh b/get_public_transports_from_France_distinct.bm.sh old mode 100644 new mode 100755 diff --git a/get_public_transports_from_France_distinct.sh b/get_public_transports_from_France_distinct.sh old mode 100644 new mode 100755 diff --git a/get_public_transports_from_Isere_distinct.sh b/get_public_transports_from_Isere_distinct.sh old mode 100644 new mode 100755 From 1d0480f98dd85ad5e0da3dcacac5293990acd2da Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Tue, 16 Sep 2025 15:43:30 +0200 Subject: [PATCH 07/41] [GHCI] source env file before launching pytest (cherry picked from commit bc12f10bf6394a81f4c8453db8c6cf19363ae53d) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b39c5a26..442304ce7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: sleep 30 # ElasticSearch takes few seconds to start, make sure it is available when the build script runs - run: curl -v http://localhost:9200 - name: Run tests - run: pytest --cov-report term --cov-report xml --cov=c2corg_api + run: export $(cat .env | grep -v "^#" | xargs) && pytest --cov-report term --cov-report xml --cov=c2corg_api - name: Send coverage to codacy # secrets are not available for PR from forks, and dependabot PRs if: ${{ github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' }} From fbaf1901fc72a564c23a9b8359f968e6ed653b42 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Fri, 10 Oct 2025 10:48:04 +0200 Subject: [PATCH 08/41] [chore] replace deprecated log.warn by log.warning --- c2corg_api/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 2d9e2cde8..9e95af18b 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -363,7 +363,7 @@ def calculate_route_duration(mapper, connection, route): jour du script bash. """ route_id = route.document_id - log.warn(f"Calculating duration for route ID: {route_id}") + log.warning(f"Calculating duration for route ID: {route_id}") # Récupération des activités et normalisation des dénivelés activities = route.activities if route.activities is not None else [] @@ -440,7 +440,7 @@ def _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_ return None # Pas de données utilisables pour le calcul dm = dp / v_diff - log.warn(f"Calculated climbing route duration for route {route_id} (activity {activity}, no difficulties_height): {dm:.2f} hours") # noqa: E501 + log.warning(f"Calculated climbing route duration for route {route_id} (activity {activity}, no difficulties_height): {dm:.2f} hours") # noqa: E501 return dm # CAS 2: Le dénivelé des difficultés est renseigné @@ -448,7 +448,7 @@ def _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_ # Vérification de cohérence if dp > 0 and d_diff > dp: - log.warn(f"Route {route_id}: Inconsistent difficulties_height ({d_diff}m) > height_diff_up ({dp}m). Returning NULL.") # noqa: E501 + log.warning(f"Route {route_id}: Inconsistent difficulties_height ({d_diff}m) > height_diff_up ({dp}m). Returning NULL.") # noqa: E501 return None # Calcul du temps des difficultés @@ -466,7 +466,7 @@ def _calculate_climbing_duration(route, height_diff_up, height_diff_down, route_ # Calcul final selon le cadrage: max(t_diff, t_app) + 0.5 * min(t_diff, t_app) # noqa: E501 dm = max(t_diff, t_app) + 0.5 * min(t_diff, t_app) - log.warn(f"Calculated climbing route duration for route {route_id} (activity {activity}): {dm:.2f} hours (t_diff={t_diff:.2f}, t_app={t_app:.2f})") # noqa: E501 + log.warning(f"Calculated climbing route duration for route {route_id} (activity {activity}): {dm:.2f} hours (t_diff={t_diff:.2f}, t_app={t_app:.2f})") # noqa: E501 return dm @@ -517,7 +517,7 @@ def _calculate_standard_duration(activity, route, height_diff_up, height_diff_do else: dm = (dv / 2) + dh - log.warn(f"Calculated standard route duration for route {route_id} (activity {activity}): {dm:.2f} hours") # noqa: E501 + log.warning(f"Calculated standard route duration for route {route_id} (activity {activity}): {dm:.2f} hours") # noqa: E501 return dm @@ -551,6 +551,6 @@ def _update_route_duration(connection, route_id, calculated_duration_in_days): ), {"duration": calculated_duration_in_days, "route_id": route_id}, ) - log.warn( + log.warning( f"Route {route_id}: Database updated with calculated_duration = {calculated_duration_in_days} days." # noqa: E501 ) From b6caaa414dc1d35b93d4548aea41c9fa65fc8067 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Fri, 10 Oct 2025 10:49:38 +0200 Subject: [PATCH 09/41] [fix] delete existing stopareas for waypoint if moved in an unserved area --- c2corg_api/__init__.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 9e95af18b..edcf77af0 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -100,6 +100,21 @@ def configure_anonymous(settings, config): account_id = int(settings.get("guidebook.anonymous_user_account")) config.registry.anonymous_user_id = account_id +def delete_waypoint_stopareas(connection, waypoint_id): + # Delete existing stopareas for waypoint + delete_relation_query = text( + """ + DELETE FROM guidebook.waypoints_stopareas + WHERE waypoint_id = :waypoint_id + """ + ) + + connection.execute( + delete_relation_query, + { + "waypoint_id": waypoint_id, + }, + ) @event.listens_for(DocumentGeometry, "after_insert") @event.listens_for(DocumentGeometry, "after_update") @@ -182,7 +197,8 @@ def process_new_waypoint(mapper, connection, geometry): places_data = places_response.json() if "places_nearby" not in places_data or not places_data["places_nearby"]: - log.warning(f"No Navitia stops found for the waypoint {waypoint_id}") + log.warning(f"No Navitia stops found for the waypoint {waypoint_id}; deleting previously registered stops") + delete_waypoint_stopareas(connection, waypoint_id) return # --- NOUVEAU : Filtrage par diversité de transport (comme dans bash) --- @@ -226,23 +242,11 @@ def process_new_waypoint(mapper, connection, geometry): known_transports.update(current_stop_transports) selected_count += 1 - # Delete existing stopareas for waypoint - delete_relation_query = text( - """ - DELETE FROM guidebook.waypoints_stopareas - WHERE waypoint_id = :waypoint_id - """ - ) - - connection.execute( - delete_relation_query, - { - "waypoint_id": waypoint_id, - }, - ) - log.warning(f"Selected {selected_count} stops out of {len(places_data['places_nearby'])} for waypoint {waypoint_id}") # noqa: E501 + log.warning(f"Deleting previously registered stops") + delete_waypoint_stopareas(connection, waypoint_id) + # Traiter uniquement les arrêts sélectionnés for place in selected_stops: stop_id = place["id"] From 6a88e1884cef6be3132595e8345ed70546385a5f Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Fri, 10 Oct 2025 10:51:08 +0200 Subject: [PATCH 10/41] [fix] formatting error in log string that was triggering 500 errors when saving (issue 1789) --- c2corg_api/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index edcf77af0..637004561 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -535,8 +535,9 @@ def _validate_and_convert_duration(min_duration, route_id): or min_duration < min_duration_hours or min_duration > max_duration_hours ): - log.warn( - f"Route {route_id}: Calculated duration ({min_duration:.2f} hours if not None) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 + min_duration_str = "None" if min_duration is None else "{min_duration:.2f}" + log.warning( + f"Route {route_id}: Calculated duration (min_duration={min_duration}) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 ) return None From 3a540a1dbf602871a984fa69125a8fece3cb0469 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Fri, 10 Oct 2025 11:53:59 +0200 Subject: [PATCH 11/41] [chore] fix linting issues --- c2corg_api/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 637004561..14e82ffd5 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -100,6 +100,7 @@ def configure_anonymous(settings, config): account_id = int(settings.get("guidebook.anonymous_user_account")) config.registry.anonymous_user_id = account_id + def delete_waypoint_stopareas(connection, waypoint_id): # Delete existing stopareas for waypoint delete_relation_query = text( @@ -116,6 +117,7 @@ def delete_waypoint_stopareas(connection, waypoint_id): }, ) + @event.listens_for(DocumentGeometry, "after_insert") @event.listens_for(DocumentGeometry, "after_update") def process_new_waypoint(mapper, connection, geometry): @@ -197,7 +199,7 @@ def process_new_waypoint(mapper, connection, geometry): places_data = places_response.json() if "places_nearby" not in places_data or not places_data["places_nearby"]: - log.warning(f"No Navitia stops found for the waypoint {waypoint_id}; deleting previously registered stops") + log.warning(f"No Navitia stops found for the waypoint {waypoint_id}; deleting previously registered stops") # noqa: E501 delete_waypoint_stopareas(connection, waypoint_id) return @@ -244,7 +246,7 @@ def process_new_waypoint(mapper, connection, geometry): log.warning(f"Selected {selected_count} stops out of {len(places_data['places_nearby'])} for waypoint {waypoint_id}") # noqa: E501 - log.warning(f"Deleting previously registered stops") + log.warning("Deleting previously registered stops") delete_waypoint_stopareas(connection, waypoint_id) # Traiter uniquement les arrêts sélectionnés @@ -535,9 +537,9 @@ def _validate_and_convert_duration(min_duration, route_id): or min_duration < min_duration_hours or min_duration > max_duration_hours ): - min_duration_str = "None" if min_duration is None else "{min_duration:.2f}" + min_duration_str = "None" if min_duration is None else f"{min_duration:.2f}" # noqa: E501 log.warning( - f"Route {route_id}: Calculated duration (min_duration={min_duration}) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 + f"Route {route_id}: Calculated duration (min_duration={min_duration_str}) is out of bounds (min={min_duration_hours}h, max={max_duration_hours}h) or NULL. Setting duration to NULL." # noqa: E501 ) return None From 8233f1d6787a5357b1f6a2b77de8eab90abea230 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Fri, 10 Oct 2025 11:54:18 +0200 Subject: [PATCH 12/41] [CI] run test script with set -e --- launch_ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/launch_ci.sh b/launch_ci.sh index 45749310a..994f4a375 100755 --- a/launch_ci.sh +++ b/launch_ci.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e apt update apt install -y postgresql-client From bb40e9e76af6350534917af7d6c80a1369420193 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 24 Nov 2025 14:34:19 +0100 Subject: [PATCH 13/41] [new feature] add reachable routes route --- .../common/sortable_search_attributes.py | 36 +++ c2corg_api/views/route.py | 278 +++++++++++++++++- 2 files changed, 313 insertions(+), 1 deletion(-) diff --git a/c2corg_api/models/common/sortable_search_attributes.py b/c2corg_api/models/common/sortable_search_attributes.py index 946d0b97b..3f3fe3608 100644 --- a/c2corg_api/models/common/sortable_search_attributes.py +++ b/c2corg_api/models/common/sortable_search_attributes.py @@ -348,3 +348,39 @@ 'slope_40_45': 3, 'slope_gt_45': 4 } + + +sortable_search_attr_by_field = { + 'quality': sortable_quality_types, + 'access_time': sortable_access_times, + 'paragliding_rating': sortable_paragliding_ratings, + 'durations': sortable_route_duration_types, + 'ski_rating': sortable_ski_ratings, + 'ski_exposition': sortable_exposition_ratings, + 'labande_ski_rating': sortable_labande_ski_ratings, + 'labande_global_rating': sortable_global_ratings, + 'global_rating': sortable_global_ratings, + 'engagement_rating': sortable_engagement_ratings, + 'risk_rating': sortable_risk_ratings, + 'equipment_rating': sortable_equipment_ratings, + 'ice_rating': sortable_ice_ratings, + 'mixed_rating': sortable_mixed_ratings, + 'exposition_rock_rating': sortable_exposition_rock_ratings, + 'rock_free_rating': sortable_climbing_ratings, + 'rock_required_rating': sortable_climbing_ratings, + 'aid_rating': sortable_aid_ratings, + 'via_ferrata_rating': sortable_via_ferrata_ratings, + 'hiking_rating': sortable_hiking_ratings, + 'hiking_mtb_exposition': sortable_exposition_ratings, + 'snowshoe_rating': sortable_snowshoe_ratings, + 'mtb_up_rating': sortable_mtb_up_ratings, + 'mtb_down_rating': sortable_mtb_down_ratings, + 'frequentation': sortable_frequentation_types, + 'condition_rating': sortable_condition_ratings, + 'snow_quality': sortable_snow_quality_ratings, + 'snow_quantity': sortable_snow_quality_ratings, + 'glacier_rating': sortable_glacier_ratings, + 'severity': sortable_severities, + 'avalanche_level': sortable_avalanche_levels, + 'avalanche_slope': sortable_avalanche_slopes +} diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 2ab7f9cb3..afae3ecd1 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -34,6 +34,33 @@ from sqlalchemy.orm import load_only from sqlalchemy.sql.expression import text, or_, column, union +from operator import and_, or_ +from c2corg_api.models.area import Area +from c2corg_api.models.area_association import AreaAssociation +from c2corg_api.models.association import Association +from c2corg_api.models.document import DocumentGeometry, DocumentLocale +from c2corg_api.models.utils import ArrayOfEnum +from c2corg_api.search.search_filters import build_query +from c2corg_api.views.validation import validate_pagination, validate_preferred_lang_param +from c2corg_api.views import cors_policy, to_json_dict +from c2corg_api.views.document import ( + LIMIT_DEFAULT, DocumentRest) +from c2corg_api.models.waypoint_stoparea import ( + WaypointStoparea) +from c2corg_api.models import DBSession +from c2corg_api.models.route import ROUTE_TYPE, Route +from c2corg_api.models.waypoint import Waypoint +from c2corg_api.models.area import schema_listing_area +from c2corg_api.models.route import schema_route +from shapely.geometry import Polygon +from geoalchemy2.shape import from_shape +from sqlalchemy import func, literal_column +from geoalchemy2.functions import ST_Intersects, ST_Transform +from cornice.resource import resource, view +from c2corg_api.models.common.sortable_search_attributes import sortable_search_attr_by_field +from sqlalchemy import nullslast + + validate_route_create = make_validator_create( fields_route, 'activities', activities) validate_route_update = make_validator_update( @@ -202,6 +229,71 @@ def get(self): update_all_pt_rating(waypoint_extrapolation) +@resource(path='/reachableroutes', cors_policy=cors_policy) +class ReachableRouteRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def get(self): + """Returns a list of object {documents: Route[], total: Integer} -> + documents: routes reachable within offset and limit + total: number of documents returned by query without offset and limit""" + + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + query = build_reachable_route_query(self.request.GET, meta_params) + + count = query.count() + + results = ( + query + .limit(meta_params['limit']) + .offset(meta_params['offset']) + .all() + ) + + areas_id = set() + for route, areas in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + routes = [] + for route, areas in results: + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + route.areas = json_areas + wp_dict = to_json_dict(route, schema_route, True) + routes.append(wp_dict) + + return {'documents': routes, 'total': count} + + def set_default_geometry(linked_waypoints, route, user_id): """When creating a new route, set the default geometry to the middle point of a given track, if not to the geometry of the associated main waypoint @@ -499,7 +591,7 @@ def worst_rating(rating1, rating2): # Return the best starting rating if it's not a crossing if not (route_types and bool( set(["traverse", "raid", "expedition"]) & set(route_types) - )): + )): return best_starting_rating # If no ending point is provided @@ -513,3 +605,187 @@ def worst_rating(rating1, rating2): # Return the worst of the two ratings return worst_rating(best_starting_rating, best_ending_rating) + + +def build_reachable_route_query(params, meta_params): + """build the query based on params and meta params. + this includes every filters on route, as well as offset + limit, sort, bbox... + returns a list of routes reachable (accessible by common transports), filtered with params + """ + search = build_query(params, meta_params, ROUTE_TYPE) + search_dict = search.to_dict() + filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) + + filter_conditions = [] + + # Mapping filter keys to models + filter_map = { + 'areas': Area, + 'waypoints': Waypoint + } + + # the array of conditions to filter the query results + filter_conditions = [] + + # the array of langs available + lang = [] + + # loop over each filter + for f in filters: + for filter_key, param in f.items(): + for param_key, param_value in param.items(): + # special cases + if param_key == 'available_locales': # lang is determined by the available locales for the document + if isinstance(param_value, list): + lang = param_value + else: + lang = [param_value] + elif param_key == 'geom': + col = getattr(DocumentGeometry, 'geom') + polygon = Polygon([ + (param_value['left'], param_value['bottom']), + (param_value['right'], param_value['bottom']), + (param_value['right'], param_value['top']), + (param_value['left'], param_value['top']), + (param_value['left'], param_value['bottom']) + ]) + polygon_wkb = from_shape(polygon, srid=4326) + + filter_conditions.append(ST_Intersects( + ST_Transform(col, 4326), polygon_wkb)) + elif param_key in filter_map: # param_key is 'area' or 'waypoint' + col = getattr(filter_map[param_key], 'document_id') + if isinstance(param_value, list): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + else: # all filters on Route + # col <=> Route.param_key + col = getattr(Route, param_key) + column = col.property.columns[0] + col_type = column.type + + if filter_key == 'range': + # lte and gte are integers + gte = param_value.get('gte') + lte = param_value.get('lte') + mapper = sortable_search_attr_by_field[param_key] + values = [] + if gte is not None and lte is not None: + if gte == lte: + values = [ + val for val in mapper if mapper[val] == gte and mapper[val] == lte] + else: + # find array of possible values (not integers but enum values) between gte and lte + values = [ + val for val in mapper if mapper[val] >= gte and mapper[val] < lte] + + elif gte is not None: + # find array of possible values (not integers but enum values) >= gte + values = [ + val for val in mapper if mapper[val] >= gte] + + elif lte is not None: + # find array of possible values (not integers but enum values) < lte + values = [ + val for val in mapper if mapper[val] < lte] + + # then compare (==) col with each value + # combine multiple checks with | + checks = [(col == val) for val in values] + if len(checks) > 0: + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + + filter_conditions.append(or_expr) + + elif filter_key == 'terms': + values = param_value if isinstance( + param_value, (list, tuple)) else [param_value] + + if isinstance(col_type, ArrayOfEnum): + # combine multiple checks with | + checks = [col.any(v) for v in values] + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + filter_conditions.append(or_expr) + else: + filter_conditions.append(col.in_(values)) + + elif filter_key == 'term': + if isinstance(col_type, ArrayOfEnum): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + + else: + continue + + # combine all conditions with & + if len(filter_conditions) == 0: + filter_conditions = True + elif len(filter_conditions) == 1: + filter_conditions = filter_conditions[0] + else: + final_expr = filter_conditions[0] + for cond in filter_conditions[1:]: + final_expr = final_expr & cond + filter_conditions = final_expr + + # get sort information + sorts = search_dict.get('sort', []) + # compute sort expressions + sort_expressions = [] + for sort in sorts: + if (sort == 'undefined'): + pass + # sort by desc + elif (hasattr(sort, 'items')): + for attribute, order in sort.items(): + if (attribute == 'id'): + sort_expressions.append( + nullslast(getattr(Route, 'document_id').desc())) + else: + sort_expressions.append( + nullslast(getattr(Route, attribute).desc())) + else: + # sort by asc + sort_expressions.append(nullslast(getattr(Route, sort).asc())) + + # perform query + query = DBSession.query(Route, func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column("'document_id'"), Area.document_id + ))).label("areas")). \ + select_from(Association). \ + join(Route, or_( + Route.document_id == Association.child_document_id, + Route.document_id == Association.parent_document_id + )). \ + join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ + join(Waypoint, or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + )). \ + filter(Waypoint.waypoint_type == 'access'). \ + join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ + join(AreaAssociation, or_( + AreaAssociation.document_id == Association.child_document_id, + AreaAssociation.document_id == Association.parent_document_id + )). \ + join(Area, Area.document_id == AreaAssociation.area_id) + + if (len(lang) > 0): + query = query.join(DocumentLocale, and_( + DocumentLocale.document_id == Route.document_id, + DocumentLocale.lang.in_(lang) + )) + query = query. \ + filter(filter_conditions). \ + order_by(*sort_expressions). \ + group_by(Route). \ + distinct() + + return query From 830581af2744d5bd75365570faae11f786f0ce8f Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 24 Nov 2025 14:34:33 +0100 Subject: [PATCH 14/41] [new feature] add reachable waypoints route --- c2corg_api/views/waypoint.py | 277 ++++++++++++++++++++++++++++++++++- 1 file changed, 272 insertions(+), 5 deletions(-) diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index a1d4ce97a..d8e212c37 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -22,7 +22,7 @@ schema_create_waypoint) from c2corg_api.views.document import ( - DocumentRest, make_validator_create, make_validator_update, + LIMIT_MAX, DocumentRest, make_validator_create, make_validator_update, NUM_RECENT_OUTINGS) from c2corg_api.views import cors_policy, restricted_json_view from c2corg_api.views.validation import validate_id, validate_pagination, \ @@ -34,6 +34,30 @@ from sqlalchemy.orm.util import aliased from sqlalchemy.sql.elements import literal_column from sqlalchemy.sql.expression import and_, text, union, column +from operator import and_, or_ +from c2corg_api.models.area import Area +from c2corg_api.models.area_association import AreaAssociation +from c2corg_api.models.association import Association +from c2corg_api.models.document import DocumentGeometry, DocumentLocale +from c2corg_api.models.utils import ArrayOfEnum +from c2corg_api.search.search_filters import build_query +from c2corg_api.views.validation import validate_pagination, validate_preferred_lang_param +from c2corg_api.views import cors_policy, to_json_dict +from c2corg_api.views.document import ( + LIMIT_DEFAULT, DocumentRest) +from c2corg_api.models.waypoint_stoparea import ( + WaypointStoparea) +from c2corg_api.models import DBSession +from c2corg_api.models.route import ROUTE_TYPE, Route +from c2corg_api.models.waypoint import Waypoint +from c2corg_api.models.area import schema_listing_area +from shapely.geometry import Polygon +from geoalchemy2.shape import from_shape +from sqlalchemy import func, literal_column +from geoalchemy2.functions import ST_Intersects, ST_Transform +from cornice.resource import resource, view +from c2corg_api.models.common.sortable_search_attributes import sortable_search_attr_by_field +from sqlalchemy import nullslast # the number of routes that are included for waypoints NUM_ROUTES = 400 @@ -287,7 +311,7 @@ def set_recent_outings(waypoint, lang): join( with_query_waypoints, with_query_waypoints.c.document_id == t_route_wp.parent_document_id - ). \ + ). \ distinct(). \ count() @@ -369,9 +393,9 @@ def _get_select_children(waypoint): cte('waypoint_grandchildren') return union( - select_waypoint.select(), - select_waypoint_children.select(), - select_waypoint_grandchildren.select()). \ + select_waypoint.select(), + select_waypoint_children.select(), + select_waypoint_grandchildren.select()). \ cte('select_all_waypoints') @@ -393,6 +417,70 @@ def get(self): return self._get_document_info(waypoint_documents_config) +@resource(path='/reachablewaypoints', cors_policy=cors_policy) +class ReachableWaypointRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def get(self): + """Returns a list of object {documents: Waypoint[], total: Integer} -> + documents: waypoints reachable within offset and limit + total: number of documents returned by query without offset and limit""" + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': min(validated.get('limit', LIMIT_DEFAULT), LIMIT_MAX), + 'lang': validated.get('lang') + } + + query = build_reachable_waypoints_query(self.request.GET, meta_params) + + count = query.count() + + results = ( + query + .limit(meta_params['limit']) + .offset(meta_params['offset']) + .all() + ) + + areas_id = set() + for wp, areas in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + waypoints = [] + for wp, areas in results: + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + wp.areas = json_areas + wp_dict = to_json_dict(wp, schema_waypoint, True) + waypoints.append(wp_dict) + + return {'documents': waypoints, 'total': count} + + def update_linked_routes(waypoint, update_types, user_id): update_linked_route_titles(waypoint, update_types, user_id) update_linked_routes_public_transportation_rating(waypoint, update_types) @@ -459,3 +547,182 @@ def update_linked_routes_public_transportation_rating(waypoint, update_types): for route in routes: update_pt_rating(route) + + +def build_reachable_waypoints_query(params, meta_params): + """build the query based on params and meta params. + this includes every filters on waypoints, as well as offset + limit, sort, bbox... + returns a list of waypoints reachable (accessible by common transports), filtered with params + """ + search = build_query(params, meta_params, ROUTE_TYPE) + search_dict = search.to_dict() + filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) + + filter_conditions = [] + + # Mapping filter keys to models + filter_map = { + 'areas': Area + } + + # the array of conditions to filter the query results + filter_conditions = [] + + # the array of langs available + lang = [] + + # loop over each filter + for f in filters: + for filter_key, param in f.items(): + for param_key, param_value in param.items(): + # special cases + if param_key == 'available_locales': # lang is determined by the available locales for the document + if isinstance(param_value, list): + lang = param_value + else: + lang = [param_value] + elif param_key == 'geom': + col = getattr(DocumentGeometry, 'geom') + polygon = Polygon([ + (param_value['left'], param_value['bottom']), + (param_value['right'], param_value['bottom']), + (param_value['right'], param_value['top']), + (param_value['left'], param_value['top']), + (param_value['left'], param_value['bottom']) + ]) + polygon_wkb = from_shape(polygon, srid=4326) + + filter_conditions.append(ST_Intersects( + ST_Transform(col, 4326), polygon_wkb)) + elif param_key in filter_map: # param_key is 'area' + col = getattr(filter_map[param_key], 'document_id') + if isinstance(param_value, list): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + else: # all filters on Waypoints + # col <=> Waypoint.param_key + col = getattr(Waypoint, param_key) + column = col.property.columns[0] + col_type = column.type + + if filter_key == 'range': + # lte and gte are integers + gte = param_value.get('gte') + lte = param_value.get('lte') + mapper = sortable_search_attr_by_field[param_key] + values = [] + if gte is not None and lte is not None: + if gte == lte: + values = [ + val for val in mapper if mapper[val] == gte and mapper[val] == lte] + else: + # find array of possible values (not integers but enum values) between gte and lte + values = [ + val for val in mapper if mapper[val] >= gte and mapper[val] < lte] + + elif gte is not None: + # find array of possible values (not integers but enum values) >= gte + values = [ + val for val in mapper if mapper[val] >= gte] + + elif lte is not None: + # find array of possible values (not integers but enum values) < lte + values = [ + val for val in mapper if mapper[val] < lte] + + # then compare (==) col with each value + # combine multiple checks with | + checks = [(col == val) for val in values] + if len(checks) > 0: + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + + filter_conditions.append(or_expr) + + elif filter_key == 'terms': + values = param_value if isinstance( + param_value, (list, tuple)) else [param_value] + + if isinstance(col_type, ArrayOfEnum): + # combine multiple checks with | + checks = [col.any(v) for v in values] + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + filter_conditions.append(or_expr) + else: + filter_conditions.append(col.in_(values)) + + elif filter_key == 'term': + if isinstance(col_type, ArrayOfEnum): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + + else: + continue + + # combine all conditions with & + if len(filter_conditions) == 0: + filter_conditions = True + elif len(filter_conditions) == 1: + filter_conditions = filter_conditions[0] + else: + final_expr = filter_conditions[0] + for cond in filter_conditions[1:]: + final_expr = final_expr & cond + filter_conditions = final_expr + + # get sort information + sorts = search_dict.get('sort', []) + # compute sort expressions + sort_expressions = [] + for sort in sorts: + if (sort == 'undefined'): + pass + # sort by desc + if (hasattr(sort, 'items')): + for attribute, order in sort.items(): + if (attribute == 'id'): + sort_expressions.append( + nullslast(getattr(Waypoint, 'document_id').desc())) + else: + sort_expressions.append( + nullslast(getattr(Waypoint, attribute).desc())) + else: + # sort by asc + sort_expressions.append(nullslast(getattr(Waypoint, sort).asc())) + + # perform query + query = DBSession.query(Waypoint, func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column("'document_id'"), Area.document_id + ))).label("areas")). \ + select_from(Association). \ + join(Waypoint, or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + )). \ + join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ + filter(Waypoint.waypoint_type == 'access'). \ + join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ + join(AreaAssociation, or_( + AreaAssociation.document_id == Association.child_document_id, + AreaAssociation.document_id == Association.parent_document_id + )). \ + join(Area, Area.document_id == AreaAssociation.area_id) + + if (len(lang) > 0): + query = query.join(DocumentLocale, and_( + DocumentLocale.document_id == Route.document_id, + DocumentLocale.lang.in_(lang) + )) + query = query. \ + filter(filter_conditions). \ + order_by(*sort_expressions). \ + group_by(Waypoint). \ + distinct() + + return query From ddb2f0327c352221b1d7b978ac3a3c525be2018d Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 24 Nov 2025 14:35:57 +0100 Subject: [PATCH 15/41] [new feature] add coverage table --- .../versions/27bf1b7197a6_add_coverages.py | 39 +++++++ c2corg_api/models/__init__.py | 3 + c2corg_api/models/common/attributes.py | 8 ++ c2corg_api/models/common/document_types.py | 3 +- c2corg_api/models/common/fields_coverage.py | 21 ++++ c2corg_api/models/coverage.py | 90 +++++++++++++++ c2corg_api/models/enums.py | 2 + c2corg_api/models/schema_utils.py | 2 + .../scripts/migration/documents/coverage.py | 88 ++++++++++++++ c2corg_api/scripts/migration/migrate.py | 2 + c2corg_api/views/coverage.py | 107 ++++++++++++++++++ c2corg_api/views/document_delete.py | 3 + c2corg_api/views/document_revert.py | 1 + c2corg_api/views/document_schemas.py | 9 +- 14 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 alembic_migration/versions/27bf1b7197a6_add_coverages.py create mode 100644 c2corg_api/models/common/fields_coverage.py create mode 100644 c2corg_api/models/coverage.py create mode 100644 c2corg_api/scripts/migration/documents/coverage.py create mode 100644 c2corg_api/views/coverage.py diff --git a/alembic_migration/versions/27bf1b7197a6_add_coverages.py b/alembic_migration/versions/27bf1b7197a6_add_coverages.py new file mode 100644 index 000000000..d068e1b83 --- /dev/null +++ b/alembic_migration/versions/27bf1b7197a6_add_coverages.py @@ -0,0 +1,39 @@ +"""Add coverages + +Revision ID: 335e0bc4df28 +Revises: 6b40cb9c7c3d +Create Date: 2025-11-18 14:15:26.377504 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '335e0bc4df28' +down_revision = '6b40cb9c7c3d' +branch_labels = None +depends_on = None + +def upgrade(): + coverage_type = sa.Enum('fr-idf', 'fr-ne', 'fr-nw', 'fr-se', 'fr-sw', name='coverage_type', schema='guidebook') + op.create_table('coverages', + sa.Column('coverage_type', coverage_type, nullable=True), + sa.Column('document_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['document_id'], ['guidebook.documents.document_id'], ), + sa.PrimaryKeyConstraint('document_id'), + schema='guidebook' + ) + op.create_table('coverages_archives', + sa.Column('coverage_type', coverage_type, nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['id'], ['guidebook.documents_archives.id'], ), + sa.PrimaryKeyConstraint('id'), + schema='guidebook' + ) + + +def downgrade(): + op.drop_table('coverages_archives', schema='guidebook') + op.drop_table('coverages', schema='guidebook') + sa.Enum('fr-idf', 'fr-ne', 'fr-nw', 'fr-se', 'fr-sw', name='coverage_type', schema='guidebook').drop(op.get_bind()) diff --git a/c2corg_api/models/__init__.py b/c2corg_api/models/__init__.py index f3f8d4b5c..e6cb04924 100644 --- a/c2corg_api/models/__init__.py +++ b/c2corg_api/models/__init__.py @@ -24,6 +24,7 @@ class BaseMixin(object): # all models, for which tables should be created, must be listed here: from c2corg_api.models import document # noqa +from c2corg_api.models import coverage # noqa from c2corg_api.models import waypoint # noqa from c2corg_api.models import route # noqa from c2corg_api.models import document_history # noqa @@ -59,6 +60,7 @@ class BaseMixin(object): topo_map.MAP_TYPE: topo_map.TopoMap, area.AREA_TYPE: area.Area, outing.OUTING_TYPE: outing.Outing, + coverage.COVERAGE_TYPE: coverage.Coverage, } document_locale_types = { @@ -72,4 +74,5 @@ class BaseMixin(object): topo_map.MAP_TYPE: document.DocumentLocale, area.AREA_TYPE: document.DocumentLocale, outing.OUTING_TYPE: outing.OutingLocale, + coverage.COVERAGE_TYPE: document.DocumentLocale, } diff --git a/c2corg_api/models/common/attributes.py b/c2corg_api/models/common/attributes.py index ecf4dafca..6f21de152 100644 --- a/c2corg_api/models/common/attributes.py +++ b/c2corg_api/models/common/attributes.py @@ -785,3 +785,11 @@ 'highline', 'waterline' ] + +coverage_types = [ + 'fr-idf', + 'fr-ne', + 'fr-nw', + 'fr-se', + 'fr-sw' +] diff --git a/c2corg_api/models/common/document_types.py b/c2corg_api/models/common/document_types.py index d7fb2a854..356674425 100644 --- a/c2corg_api/models/common/document_types.py +++ b/c2corg_api/models/common/document_types.py @@ -9,8 +9,9 @@ WAYPOINT_TYPE = 'w' BOOK_TYPE = 'b' XREPORT_TYPE = 'x' +COVERAGE_TYPE = 'z' ALL = [ AREA_TYPE, ARTICLE_TYPE, IMAGE_TYPE, MAP_TYPE, OUTING_TYPE, ROUTE_TYPE, - USERPROFILE_TYPE, WAYPOINT_TYPE, BOOK_TYPE, XREPORT_TYPE + USERPROFILE_TYPE, WAYPOINT_TYPE, BOOK_TYPE, XREPORT_TYPE, COVERAGE_TYPE ] diff --git a/c2corg_api/models/common/fields_coverage.py b/c2corg_api/models/common/fields_coverage.py new file mode 100644 index 000000000..4aaacfee9 --- /dev/null +++ b/c2corg_api/models/common/fields_coverage.py @@ -0,0 +1,21 @@ +DEFAULT_FIELDS = [ + 'coverage_type' + 'geometry.geom_detail' +] + +DEFAULT_REQUIRED = [ + 'coverage_type', + 'geometry', + 'geometry.geom_detail' +] + +LISTING_FIELDS = [ + 'coverage_type', + 'geometry.geom_detail' +] + +fields_coverage = { + 'fields': DEFAULT_FIELDS, + 'required': DEFAULT_REQUIRED, + 'listing': LISTING_FIELDS +} diff --git a/c2corg_api/models/coverage.py b/c2corg_api/models/coverage.py new file mode 100644 index 000000000..6fe17d861 --- /dev/null +++ b/c2corg_api/models/coverage.py @@ -0,0 +1,90 @@ +from c2corg_api.models import Base, schema +from c2corg_api.models.enums import coverage_types +from c2corg_api.models.document import ( + ArchiveDocument, Document, get_geometry_schema_overrides, schema_document_locale, schema_attributes) +from c2corg_api.models.schema_utils import get_update_schema, \ + get_create_schema, restrict_schema +from c2corg_api.models.utils import copy_attributes +from c2corg_api.models.common.fields_coverage import fields_coverage +from colanderalchemy import SQLAlchemySchemaNode +from sqlalchemy import ( + Column, + Integer, + ForeignKey +) +from c2corg_api.models.common import document_types + +COVERAGE_TYPE = document_types.COVERAGE_TYPE + + +class _CoverageMixin(object): + coverage_type = Column(coverage_types) + + +attributes = ['coverage_type'] + + +class Coverage(_CoverageMixin, Document): + __tablename__ = 'coverages' + + document_id = Column( + Integer, + ForeignKey(schema + '.documents.document_id'), primary_key=True) + + __mapper_args__ = { + 'polymorphic_identity': COVERAGE_TYPE, + 'inherit_condition': Document.document_id == document_id + } + + def to_archive(self): + coverage = ArchiveCoverage() + super(Coverage, self)._to_archive(coverage) + copy_attributes(self, coverage, attributes) + + return coverage + + def update(self, other): + super(Coverage, self).update(other) + copy_attributes(other, self, attributes) + + +schema_coverage_locale = schema_document_locale +schema_coverage_attributes = list(schema_attributes) + +class ArchiveCoverage(_CoverageMixin, ArchiveDocument): + """ + """ + __tablename__ = 'coverages_archives' + + id = Column( + Integer, + ForeignKey(schema + '.documents_archives.id'), primary_key=True) + + __mapper_args__ = { + 'polymorphic_identity': COVERAGE_TYPE, + 'inherit_condition': ArchiveDocument.id == id + } + + __table_args__ = Base.__table_args__ + +schema_coverage = SQLAlchemySchemaNode( + Coverage, + # whitelisted attributes + includes=schema_coverage_attributes + attributes, + overrides={ + 'document_id': { + 'missing': None + }, + 'version': { + 'missing': None + }, + 'locales': { + 'children': [schema_coverage_locale] + }, + 'geometry': get_geometry_schema_overrides(['POLYGON']) + }) + +schema_create_coverage = get_create_schema(schema_coverage) +schema_update_coverage = get_update_schema(schema_coverage) +schema_listing_coverage = restrict_schema( + schema_coverage, fields_coverage.get('listing')) diff --git a/c2corg_api/models/enums.py b/c2corg_api/models/enums.py index e0eb5a6ac..b57469310 100644 --- a/c2corg_api/models/enums.py +++ b/c2corg_api/models/enums.py @@ -159,3 +159,5 @@ def enum(name, types): 'snow_quality_ratings', attributes.snow_quality_ratings) snow_quantity_ratings = enum( 'snow_quantity_ratings', attributes.snow_quantity_ratings) +coverage_types = enum( + 'coverage_types', attributes.coverage_types) diff --git a/c2corg_api/models/schema_utils.py b/c2corg_api/models/schema_utils.py index 8f30dcd26..3871df4d6 100644 --- a/c2corg_api/models/schema_utils.py +++ b/c2corg_api/models/schema_utils.py @@ -89,6 +89,8 @@ class SchemaAssociations(MappingSchema): Sequence(), SchemaAssociationDoc(), missing=None) outings = SchemaNode( Sequence(), SchemaAssociationDoc(), missing=None) + coverages = SchemaNode( + Sequence(), SchemaAssociationDoc(), missing=None) def get_create_schema(document_schema): diff --git a/c2corg_api/scripts/migration/documents/coverage.py b/c2corg_api/scripts/migration/documents/coverage.py new file mode 100644 index 000000000..94b403917 --- /dev/null +++ b/c2corg_api/scripts/migration/documents/coverage.py @@ -0,0 +1,88 @@ +from c2corg_api.models import enums +from c2corg_api.models.coverage import ArchiveCoverage, Coverage, COVERAGE_TYPE +from c2corg_api.models.document import DocumentLocale, ArchiveDocumentLocale, \ + DOCUMENT_TYPE +from c2corg_api.scripts.migration.documents.document import MigrateDocuments, \ + DEFAULT_QUALITY + + +class MigrateCoverages(MigrateDocuments): + + def get_name(self): + return 'coverages' + + def get_model_document(self, locales): + return DocumentLocale if locales else Coverage + + def get_model_archive_document(self, locales): + return ArchiveDocumentLocale if locales else ArchiveCoverage + + def get_document_geometry(self, document_in, version): + return dict( + document_id=document_in.id, + id=document_in.id, + version=version, + geom_detail=document_in.geom + ) + + def get_count_query(self): + return ( + 'select count(*) ' + 'from app_coverages_archives aa join coverages a on aa.id = a.id ' + 'where a.redirects_to is null;' + ) + + def get_query(self): + return ( + 'select ' + ' aa.id, aa.document_archive_id, aa.is_latest_version, ' + ' aa.is_protected, aa.redirects_to, ' + ' ST_Force2D(ST_SetSRID(aa.geom, 3857)) geom, ' + ' aa.coverage_type ' + 'from app_coverages_archives aa join coverages a on aa.id = a.id ' + 'where a.redirects_to is null ' + 'order by aa.id, aa.document_archive_id;' + ) + + def get_count_query_locales(self): + return ( + 'select count(*) ' + 'from app_coverages_i18n_archives aa join coverages a on aa.id = a.id ' + 'where a.redirects_to is null;' + ) + + def get_query_locales(self): + return ( + 'select ' + ' aa.id, aa.document_i18n_archive_id, aa.is_latest_version, ' + ' aa.culture, aa.name, aa.description ' + 'from app_coverages_i18n_archives aa join coverages a on aa.id = a.id ' + 'where a.redirects_to is null ' + 'order by aa.id, aa.culture, aa.document_i18n_archive_id;' + ) + + def get_document(self, document_in, version): + return dict( + document_id=document_in.id, + type=COVERAGE_TYPE, + version=version, + protected=document_in.is_protected, + redirects_to=document_in.redirects_to, + coverage_type=self.convert_type( + document_in.coverage_type, enums.coverage_types), + quality=DEFAULT_QUALITY + ) + + def get_document_locale(self, document_in, version): + description = self.convert_tags(document_in.description) + description, summary = self.extract_summary(description) + return dict( + document_id=document_in.id, + id=document_in.document_i18n_archive_id, + type=DOCUMENT_TYPE, + version=version, + lang=document_in.culture, + title=document_in.name, + description=description, + summary=summary + ) \ No newline at end of file diff --git a/c2corg_api/scripts/migration/migrate.py b/c2corg_api/scripts/migration/migrate.py index 818a03295..9c8b2bdd3 100644 --- a/c2corg_api/scripts/migration/migrate.py +++ b/c2corg_api/scripts/migration/migrate.py @@ -6,6 +6,7 @@ MigrateAreaAssociations from c2corg_api.scripts.migration.climbing_site_routes import \ CreateClimbingSiteRoutes +from c2corg_api.scripts.migration.documents.coverage import MigrateCoverages from c2corg_api.scripts.migration.documents.xreports import MigrateXreports from c2corg_api.scripts.migration.documents.area import MigrateAreas from c2corg_api.scripts.migration.documents.associations import \ @@ -97,6 +98,7 @@ def main(argv=sys.argv): MigrateBooks(connection_source, session, batch_size).migrate() MigrateVersions(connection_source, session, batch_size).migrate() MigrateAssociations(connection_source, session, batch_size).migrate() + MigrateCoverages(connection_source, session, batch_size).migrate() CreateClimbingSiteRoutes(connection_source, session, batch_size).migrate() SetRouteTitlePrefix(connection_source, session, batch_size).migrate() SetDefaultGeometries(connection_source, session, batch_size).migrate() diff --git a/c2corg_api/views/coverage.py b/c2corg_api/views/coverage.py new file mode 100644 index 000000000..ce29df3b0 --- /dev/null +++ b/c2corg_api/views/coverage.py @@ -0,0 +1,107 @@ + +import functools +import logging +from operator import and_, or_ +from c2corg_api.models import DBSession, coverage +from c2corg_api.models.association import Association +from c2corg_api.models.common.fields_coverage import fields_coverage +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, schema_coverage, schema_create_coverage, schema_update_coverage +from c2corg_api.models.document import DocumentGeometry +from c2corg_api.models.utils import wkb_to_shape +from c2corg_api.models.waypoint import Waypoint +from c2corg_api.views.document_schemas import coverage_documents_config +from c2corg_api.views.area import update_associations +from c2corg_api.views.validation import validate_cook_param, validate_id, validate_lang_param +from c2corg_api.views.document import DocumentRest, make_validator_create, make_validator_update +from shapely import wkb +from shapely.geometry import Point +from cornice.validators import colander_body_validator +from cornice.resource import resource, view +from sqlalchemy import func + +from c2corg_api.views import cors_policy, restricted_json_view, to_json_dict +from c2corg_api.views.validation import validate_associations, validate_pagination, \ + validate_preferred_lang_param + +log = logging.getLogger(__name__) + +validate_coverage_create = make_validator_create( + fields_coverage.get('required')) +validate_coverage_update = make_validator_update( + fields_coverage.get('required')) +validate_associations_create = functools.partial( + validate_associations, COVERAGE_TYPE, True) +validate_associations_update = functools.partial( + validate_associations, COVERAGE_TYPE, False) + + +@resource(collection_path='/coverages', path='/coverages/{id}', + cors_policy=cors_policy) +class CoverageRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def collection_get(self): + return self._collection_get(COVERAGE_TYPE, coverage_documents_config) + + @view(validators=[validate_id, validate_lang_param, validate_cook_param]) + def get(self): + return self._get( + coverage_documents_config, schema_coverage, include_areas=False) + + @restricted_json_view( + schema=schema_create_coverage, + validators=[ + colander_body_validator, + validate_coverage_create, + validate_associations_create]) + def collection_post(self): + return self._collection_post(schema_coverage, allow_anonymous=False) + + @restricted_json_view( + schema=schema_update_coverage, + validators=[ + colander_body_validator, + validate_id, + validate_coverage_update, + validate_associations_update]) + def put(self): + return self._put(Coverage, schema_coverage) + + +@resource(path='/getcoverage', cors_policy=cors_policy) +class WaypointCoverageRest(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[validate_pagination, validate_preferred_lang_param]) + def get(self): + """Returns the coverage from a longitude and a latitude""" + + lon = float(self.request.GET['lon']) + lat = float(self.request.GET['lat']) + + pt = Point(lon, lat) + + coverageFound = None + + coverages = DBSession.query(Coverage).all() + + for coverage in coverages: + geom = coverage.geometry.geom_detail + + # convert WKB → Shapely polygon + poly = wkb_to_shape(geom) + + if poly.contains(pt) or poly.intersects(pt): + coverageFound = coverage + break + + if (coverageFound): + return coverageFound.coverage_type + else: + return "" + diff --git a/c2corg_api/views/document_delete.py b/c2corg_api/views/document_delete.py index 6d557f6df..107b42710 100644 --- a/c2corg_api/views/document_delete.py +++ b/c2corg_api/views/document_delete.py @@ -1,3 +1,4 @@ +from c2corg_api.models.coverage import COVERAGE_TYPE, ArchiveCoverage, Coverage from c2corg_api.security.acl import ACLDefault from c2corg_api.models import DBSession, article, image from c2corg_api.models.area_association import AreaAssociation @@ -352,6 +353,8 @@ def _get_models(document_type): return Article, None, ArchiveArticle, None if document_type == BOOK_TYPE: return Book, None, ArchiveBook, None + if document_type == COVERAGE_TYPE: + return Coverage, None, ArchiveCoverage, None if document_type == XREPORT_TYPE: return Xreport, XreportLocale, ArchiveXreport, ArchiveXreportLocale assert False diff --git a/c2corg_api/views/document_revert.py b/c2corg_api/views/document_revert.py index 7eb0be4ab..5f8243060 100644 --- a/c2corg_api/views/document_revert.py +++ b/c2corg_api/views/document_revert.py @@ -1,5 +1,6 @@ import logging +from c2corg_api.models.coverage import COVERAGE_TYPE, ArchiveCoverage, Coverage from c2corg_api.security.acl import ACLDefault from c2corg_api import DBSession from c2corg_api.models.area import AREA_TYPE, Area, ArchiveArea diff --git a/c2corg_api/views/document_schemas.py b/c2corg_api/views/document_schemas.py index db08b12b9..4f1865786 100644 --- a/c2corg_api/views/document_schemas.py +++ b/c2corg_api/views/document_schemas.py @@ -2,6 +2,7 @@ from c2corg_api.models.article import ARTICLE_TYPE, Article, \ schema_listing_article from c2corg_api.models.book import BOOK_TYPE, Book, schema_listing_book +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, schema_listing_coverage from c2corg_api.models.image import IMAGE_TYPE, Image, schema_listing_image from c2corg_api.models.outing import OUTING_TYPE, Outing, schema_outing from c2corg_api.models.xreport import XREPORT_TYPE, Xreport, \ @@ -27,6 +28,7 @@ from c2corg_api.models.common.fields_image import fields_image from c2corg_api.models.common.fields_topo_map import fields_topo_map from c2corg_api.models.common.fields_user_profile import fields_user_profile +from c2corg_api.models.common.fields_coverage import fields_coverage from functools import lru_cache @@ -253,6 +255,10 @@ def adapt_waypoint_schema_for_type(waypoint_type, field_list_type): WAYPOINT_TYPE, Waypoint, schema_waypoint, clazz_locale=WaypointLocale, fields=fields_waypoint, adapt_schema=waypoint_listing_schema_adaptor) +# coverages +coverage_documents_config = GetDocumentsConfig( + COVERAGE_TYPE, Coverage, schema_listing_coverage, + listing_fields=fields_coverage['listing']) document_configs = { WAYPOINT_TYPE: waypoint_documents_config, @@ -264,5 +270,6 @@ def adapt_waypoint_schema_for_type(waypoint_type, field_list_type): XREPORT_TYPE: xreport_documents_config, MAP_TYPE: topo_map_documents_config, ARTICLE_TYPE: article_documents_config, - USERPROFILE_TYPE: user_profile_documents_config + USERPROFILE_TYPE: user_profile_documents_config, + COVERAGE_TYPE: coverage_documents_config } From bcb9a9704c7a4cbeb4ceba0ea97a536c2d972383 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 24 Nov 2025 14:36:20 +0100 Subject: [PATCH 16/41] [new feature] add script to retrieve and insert coverages --- update_navitia_coverage.sh | 186 +++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100755 update_navitia_coverage.sh diff --git a/update_navitia_coverage.sh b/update_navitia_coverage.sh new file mode 100755 index 000000000..e1a88327b --- /dev/null +++ b/update_navitia_coverage.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# shellcheck disable=SC2001 + +# this scripts is meant to be executed whenever navitia france coverages are updated. +# the goal of this script is to save in DB france coverages, with their id and geometry. +# there are as many Navitia request as there are coverages in France. +# +# The script takes in parameter +# Username 'user123' +# Password 'password123' (make sure to escape special characters) +# Base API URL 'http://localhost' +# API Port '6543' +# And is meant to be used by moderators, as regular users can't delete documents. + +# First, a token is retrieved using username and password in parameter. +# Then, existing coverages are deleted +# Navitia request are made towards /coverages/{region_id} route to get all coverages. +# For each coverage found, a POST on Base_API_URL:API_PORT/coverages is made to insert in database. + +# NOTE: the geometry returned by Navitia for the coverages are in WGS384. + +if [ -f ./.env ]; then + # Load .env data + export $(grep -v '^#' ./.env | xargs) +else + echo ".env file not found!" + exit 1 +fi + +# Function to display usage +usage() { + echo "Usage: $0 " + exit 1 +} + +# Check if exactly 4 arguments are provided +if [ "$#" -ne 4 ]; then + usage +fi + +# Check if jq is installed +if ! command -v jq &> /dev/null; then + echo "jq could not be found. Please install it." + exit 1 +fi + +# Assign arguments to variables +username="$1" +password="$2" +base_api_url="$3" +api_port="$4" + +LOG_FILE="log-navitia-coverage.txt" +NAVITIA_REQUEST_COUNT=0 + +COVERAGE_API_URL="$base_api_url:$api_port/coverages" + +echo "Start time :" > "$LOG_FILE" +echo $(date +"%Y-%m-%d-%H-%M-%S") >> "$LOG_FILE" + +login_body=$(jq -n \ + --arg username "$username" \ + --arg password "$password" \ + '{ + username: $username, + password: $password, + discourse: true, + }') + +# log in to execute script +loginResponse=$(curl -s -X POST "$base_api_url:$api_port/users/login" \ + -H "Content-Type: application/json" \ + -d "$login_body") + +roles=$(echo "$loginResponse" | jq -r '.roles') +JWTToken=$(echo "$loginResponse" | jq -r '.token') + +coverages=$(curl -s -X GET "$COVERAGE_API_URL" \ + -H "Content-Type: application/json") + +numberOfCoverage=$(echo "$coverages" | jq -r '.total') + +if [ "$numberOfCoverage" != "0" ]; then + # check if logged user is a moderator + found=false + for role in "${roles[@]}"; do + if [[ "$role" == "moderator" ]]; then + found=true + break + fi + done + if ! $found; then + echo "Error : User should be a moderator to delete existing coverages" + exit 1 + fi + + # remove old coverages + echo "$coverages" | jq -c '.documents[]' | while IFS= read -r coverage; do + coverage_doc_id=$(echo "$coverage" | jq -r '.document_id') + + deleteResponse=$(curl -X POST -v -H "Content-Type: application/json" -H "Authorization: JWT token=\"$JWTToken\"" "$base_api_url:$api_port/documents/delete/${coverage_doc_id}") + + status=$(echo "$deleteResponse" | jq -r '.status') + + # if we can't delete coverage, then we stop the script + if [ $status = "error" ]; then + exit 1 + fi + done +fi + +# This define how much navitia request will be made executing this script +regions=('fr-idf' 'fr-ne' 'fr-nw' 'fr-se' 'fr-sw') + +responses=() + +# Loop over Navitia regions in France +for region_id in "${regions[@]}"; do + # Fetch the response from the Navitia API + response=$(curl -s -H "Authorization: $NAVITIA_API_KEY" \ + "https://api.navitia.io/v1/coverage/${region_id}") + ((NAVITIA_REQUEST_COUNT++)) + + # Extract the coverage type + coverage_type=$(echo "$response" | jq -r '.regions[0].id') + + # Extract the shape (WKT string) + shape=$(echo "$response" | jq -r '.regions[0].shape') + + # remove 'MULTIPOLGYON' from shape + coordinate_list=${shape//"MULTIPOLYGON"/} + + # remove ( + coordinate_list=${coordinate_list//"("/} + + # remove ) + coordinate_list=${coordinate_list//")"/} + + coordinates=() + coordinates+="[[" + + # get a list of all coordinates (separated by comma) + while IFS=',' read -ra coo; do + for i in "${coo[@]}"; do + # get lon & lat + lon_lat=($i) + # fix subcoordinates + operator concatenate not exist + subcoordinates="[${lon_lat[0]},${lon_lat[1]}]" + coordinates+="${subcoordinates}," + done + done <<< "$coordinate_list" + + # remove last comma + coordinates=${coordinates%?} + + coordinates+="]]" + + type="Polygon" + + geom_detail="{\"type\": \"Polygon\", \"coordinates\": $coordinates}" + + # no coverages yet, so we insert + if [ "$numberOfCoverage" = "0" ]; then + echo "inserting coverages" + # Build JSON payload + payload=$(jq -n \ + --arg coverage_type "$coverage_type" \ + --arg geom_detail "$geom_detail" \ + '{ + coverage_type: $coverage_type, + geometry: { + geom: null, + geom_detail: $geom_detail + } + }') + + # Send the POST request to create a coverage in DB + responses+=$(curl -X POST -v -H "Content-Type: application/json" -H "Authorization: JWT token=\"$JWTToken\"" -d "$payload" "$COVERAGE_API_URL") + fi +done + +# Log final progress +echo "Completed. Total Navitia API requests: $NAVITIA_REQUEST_COUNT" >> $LOG_FILE + +echo "Stop time :" >> $LOG_FILE +echo $(date +"%Y-%m-%d-%H-%M-%S") >> $LOG_FILE \ No newline at end of file From e28025a3f98b30a0d95e0afadb74ead25c572396 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Fri, 28 Nov 2025 10:29:26 +0100 Subject: [PATCH 17/41] [fix] fixed a bug when trying to insert doc that is not a waypoint --- c2corg_api/__init__.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 14e82ffd5..b08c20bf5 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -123,21 +123,9 @@ def delete_waypoint_stopareas(connection, waypoint_id): def process_new_waypoint(mapper, connection, geometry): """Processes a new waypoint to find its public transports after inserting it into documents_geometries.""" - log.debug("Entering process_new_waypoint callback") - waypoint_id = geometry.document_id - - max_distance_waypoint_to_stoparea = int( - os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") - ) - walking_speed = float(os.getenv("WALKING_SPEED")) - max_stop_area_for_1_waypoint = int(os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT")) # noqa: E501 - api_key = os.getenv("NAVITIA_API_KEY") - max_duration = int(max_distance_waypoint_to_stoparea / walking_speed) - - # Augmenter le nombre d'arrêts récupérés pour avoir plus de choix (comme dans le bash) # noqa: E501 - max_stop_area_fetched = max_stop_area_for_1_waypoint * 3 - # Check if document is a waypoint + waypoint_id = geometry.document_id + document_type = connection.execute( text( """ @@ -150,6 +138,18 @@ def process_new_waypoint(mapper, connection, geometry): if document_type != "w": return + + log.debug("Entering process_new_waypoint callback") + max_distance_waypoint_to_stoparea = int( + os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + ) + walking_speed = float(os.getenv("WALKING_SPEED")) + max_stop_area_for_1_waypoint = int(os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT")) # noqa: E501 + api_key = os.getenv("NAVITIA_API_KEY") + max_duration = int(max_distance_waypoint_to_stoparea / walking_speed) + + # Augmenter le nombre d'arrêts récupérés pour avoir plus de choix (comme dans le bash) # noqa: E501 + max_stop_area_fetched = max_stop_area_for_1_waypoint * 3 waypoint_type = connection.execute( text( From 94e05a3670a64745c6ab3736f45f4326e75bbd1f Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Fri, 28 Nov 2025 10:31:26 +0100 Subject: [PATCH 18/41] [new feature] added routes to get journey/isochrone reachable doc --- c2corg_api/views/coverage.py | 36 +-- c2corg_api/views/navitia.py | 472 ++++++++++++++++++++++++++++++++++- c2corg_api/views/route.py | 192 +++++++++++++- c2corg_api/views/waypoint.py | 71 +++--- 4 files changed, 722 insertions(+), 49 deletions(-) diff --git a/c2corg_api/views/coverage.py b/c2corg_api/views/coverage.py index ce29df3b0..bc784655a 100644 --- a/c2corg_api/views/coverage.py +++ b/c2corg_api/views/coverage.py @@ -84,24 +84,26 @@ def get(self): lon = float(self.request.GET['lon']) lat = float(self.request.GET['lat']) - pt = Point(lon, lat) + return get_coverage(lon, lat) - coverageFound = None +def get_coverage(lon, lat): + pt = Point(lon, lat) - coverages = DBSession.query(Coverage).all() + coverageFound = None - for coverage in coverages: - geom = coverage.geometry.geom_detail - - # convert WKB → Shapely polygon - poly = wkb_to_shape(geom) - - if poly.contains(pt) or poly.intersects(pt): - coverageFound = coverage - break - - if (coverageFound): - return coverageFound.coverage_type - else: - return "" + coverages = DBSession.query(Coverage).all() + for coverage in coverages: + geom = coverage.geometry.geom_detail + + # convert WKB → Shapely polygon + poly = wkb_to_shape(geom) + + if poly.contains(pt): + coverageFound = coverage + break + + if (coverageFound): + return coverageFound.coverage_type + else: + return None \ No newline at end of file diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index aaa44f7bd..16a03ece0 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -1,9 +1,27 @@ +import json +import logging import os import requests +from c2corg_api.models import DBSession +from c2corg_api.models.area import Area, schema_area +from c2corg_api.models.coverage import Coverage +from c2corg_api.models.utils import wkb_to_shape +from c2corg_api.models.waypoint import Waypoint, schema_waypoint +from c2corg_api.views.coverage import get_coverage +from c2corg_api.views.document import LIMIT_DEFAULT +from c2corg_api.views.waypoint import build_reachable_waypoints_query +from c2corg_api.views.route import build_reachable_route_query, build_reachable_route_query_with_waypoints +from shapely import wkb +from shapely.geometry import Point from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError # noqa: E501 from cornice.resource import resource, view -from c2corg_api.views import cors_policy +from c2corg_api.views import cors_policy, to_json_dict +from c2corg_api.models.route import Route, schema_route +from c2corg_api.models.area import schema_listing_area +from shapely.geometry import shape +from pyproj import Transformer +log = logging.getLogger(__name__) def validate_navitia_params(request, **kwargs): """Validates the required parameters for the Navitia API""" @@ -105,3 +123,455 @@ def get(self): raise HTTPInternalServerError(f'Network error: {str(e)}') except Exception as e: raise HTTPInternalServerError(f'Internal error: {str(e)}') + + +@resource(path='/navitia/journeyreachableroutes', cors_policy=cors_policy) +class NavitiaJourneyReachableRoutesRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def get(self): + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + journey_params = { + 'from': self.request.params.get('from'), + 'datetime': self.request.params.get('datetime'), + 'datetime_represents': self.request.params.get('datetime_represents'), + 'walking_speed': self.request.params.get('walking_speed'), + 'max_walking_duration_to_pt': self.request.params.get('max_walking_duration_to_pt'), + 'to': '' + } + + query = build_reachable_route_query_with_waypoints( + self.request.GET, meta_params) + + results = ( + query.all() + ) + + # manage areas for routes + areas_id = set() + for route, areas, waypoints in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + # manage waypoints + waypoints_id = set() + for route, areas, waypoints in results: + if waypoints is None: + return + + for waypoint in waypoints: + wp_id = waypoint.get("document_id") + if wp_id is not None: + waypoints_id.add(wp_id) + + wp_objects = DBSession.query(Waypoint).filter( + Waypoint.document_id.in_(waypoints_id)).all() + + log.warning("Number of NAVITIA journey queries : %d", len(wp_objects)) + + navitia_wp_map = {wp.document_id: is_wp_journey_reachable( + to_json_dict(wp, schema_waypoint), journey_params) for wp in wp_objects} + + routes = [] + for route, areas, waypoints in results: + # check if a journey exists for route (at least one wp has a journey associated) + journey_exists = False + for wp in waypoints: + wp_id = wp.get("document_id") + journey_exists |= navitia_wp_map.get(wp_id) + + if journey_exists: + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + route.areas = json_areas + wp_dict = to_json_dict(route, schema_route, True) + routes.append(wp_dict) + + return {'documents': routes, 'total': len(routes)} + + +@resource(path='/navitia/journeyreachablewaypoints', cors_policy=cors_policy) +class NavitiaJourneyReachableWaypointsRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def get(self): + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + journey_params = { + 'from': self.request.params.get('from'), + 'datetime': self.request.params.get('datetime'), + 'datetime_represents': self.request.params.get('datetime_represents'), + 'walking_speed': self.request.params.get('walking_speed'), + 'max_walking_duration_to_pt': self.request.params.get('max_walking_duration_to_pt'), + 'to': '' + } + + query = build_reachable_waypoints_query( + self.request.GET, meta_params) + + results = ( + query.all() + ) + + # manage areas for waypoints + areas_id = set() + for waypoints, areas in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + log.warning("Number of NAVITIA journey queries : %d", len(results)) + + waypoints = [] + for waypoint, areas in results: + # check if a journey exists for waypoint + if is_wp_journey_reachable(to_json_dict(waypoint, schema_waypoint), journey_params): + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + waypoint.areas = json_areas + wp_dict = to_json_dict(waypoint, schema_waypoint, True) + waypoints.append(wp_dict) + + return {'documents': waypoints, 'total': len(waypoints)} + +@resource(path='/navitia/isochronesreachableroutes', cors_policy=cors_policy) +class NavitiaIsochronesReachableRoutesRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def get(self): + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + isochrone_params = { + 'from': self.request.params.get('from'), + 'datetime': self.request.params.get('datetime'), + 'boundary_duration[]': self.request.params.get('boundary_duration'), + 'datetime_represents': self.request.params.get('datetime_represents') + } + + query = build_reachable_route_query_with_waypoints( + self.request.GET, meta_params) + + results = query.all() + + # manage areas for routes + areas_id = set() + for route, areas, waypoints in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + response = get_navitia_isochrone(isochrone_params) + + routes = [] + geojson = "" + # if isochrone found + if (len(response["isochrones"]) > 0): + geojson = response["isochrones"][0]["geojson"] + isochrone_geom = shape(geojson) + + # manage waypoints + waypoints_id = set() + for route, areas, waypoints in results: + if waypoints is None: + return + + for waypoint in waypoints: + wp_id = waypoint.get("document_id") + if wp_id is not None: + waypoints_id.add(wp_id) + + wp_objects = DBSession.query(Waypoint).filter( + Waypoint.document_id.in_(waypoints_id)).all() + + navitia_wp_map = {wp.document_id: is_wp_in_isochrone( + to_json_dict(wp, schema_waypoint), isochrone_geom) for wp in wp_objects} + + for route, areas, waypoints in results: + # check if a journey exists for route (at least one wp has a journey associated) + one_wp_in_isochrone = False + for wp in waypoints: + wp_id = wp.get("document_id") + one_wp_in_isochrone |= navitia_wp_map.get(wp_id) + + if one_wp_in_isochrone: + json_areas = [] + + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + route.areas = json_areas + route_dict = to_json_dict(route, schema_route, True) + routes.append(route_dict) + + return {'documents': routes, 'total': len(routes), 'isochron_geom': geojson} + + +@resource(path='/navitia/isochronesreachablewaypoints', cors_policy=cors_policy) +class NavitiaIsochronesReachableWaypointsRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def get(self): + validated = self.request.validated + + meta_params = { + 'offset': validated.get('offset', 0), + 'limit': validated.get('limit', LIMIT_DEFAULT), + 'lang': validated.get('lang') + } + + isochrone_params = { + 'from': self.request.params.get('from'), + 'datetime': self.request.params.get('datetime'), + 'boundary_duration[]': self.request.params.get('boundary_duration'), + 'datetime_represents': self.request.params.get('datetime_represents') + } + + query = build_reachable_waypoints_query( + self.request.GET, meta_params) + + results = query.all() + + # manage areas for waypoints + areas_id = set() + for waypoints, areas in results: + if areas is None: + continue + for area in areas: + area_id = area.get("document_id") + if area_id is not None: + areas_id.add(area_id) + + areas_objects = DBSession.query(Area).filter( + Area.document_id.in_(areas_id)).all() + + areas_map = {area.document_id: area for area in areas_objects} + + response = get_navitia_isochrone(isochrone_params) + + waypoints = [] + geojson = "" + # if isochrone found + if (len(response["isochrones"]) > 0): + geojson = response["isochrones"][0]["geojson"] + isochrone_geom = shape(geojson) + + for waypoint, areas in results: + # check if wp is in isochrone + if is_wp_in_isochrone(to_json_dict(waypoint, schema_waypoint), isochrone_geom): + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + waypoint.areas = json_areas + wp_dict = to_json_dict(waypoint, schema_waypoint, True) + waypoints.append(wp_dict) + + return {'documents': waypoints, 'total': len(waypoints), "isochron_geom": geojson} + + +def is_wp_journey_reachable(waypoint, journey_params): + # enhance journey params with the 'to' parameter, from the waypoint geometry. + geom = shape(json.loads(waypoint.get("geometry").get("geom"))) + + src_epsg = 3857 + transformer = Transformer.from_crs( + f"EPSG:{src_epsg}", "EPSG:4326", always_xy=True) + lon, lat = transformer.transform(geom.x, geom.y) + + journey_params['to'] = f"{lon};{lat}" + + destination_coverage = get_coverage(lon, lat) + + try: + # Récupération de la clé API depuis les variables d'environnement + api_key = os.getenv('NAVITIA_API_KEY') + if not api_key: + return False + + response = {} + + if (destination_coverage): + # Appel à l'API Navitia Journey with coverage + response = requests.get( + f'https://api.navitia.io/v1/coverage/{destination_coverage}/journeys', + params=journey_params, + headers={'Authorization': api_key}, + timeout=30 + ) + else: + # Appel à l'API Navitia Journey + response = requests.get( + 'https://api.navitia.io/v1/journeys', + params=journey_params, + headers={'Authorization': api_key}, + timeout=30 + ) + + # Vérification du statut de la réponse + if response.status_code == 401: + return False + elif response.status_code == 400: + return False + elif response.status_code == 404: + return False + elif not response.ok: + return False + + # Retour des données JSON + #TODO : return au moins 1 journey pour qui le jour de la date de départ (departure_date_time) est le même que le jour dans journey_params.date_time + return True + + except requests.exceptions.Timeout: + return False + except requests.exceptions.RequestException as e: + return False + except Exception as e: + return False + + +def get_navitia_isochrone(isochrone_params): + lon = isochrone_params.get("from").split(";")[0] + lat = isochrone_params.get("from").split(";")[1] + source_coverage = get_coverage(lon, lat) + + try: + # Récupération de la clé API depuis les variables d'environnement + api_key = os.getenv('NAVITIA_API_KEY') + if not api_key: + raise HTTPInternalServerError( + 'Configuration API Navitia manquante') + + response = {} + + if (source_coverage): + # Appel à l'API Navitia Journey with coverage + response = requests.get( + f'https://api.navitia.io/v1/coverage/{source_coverage}/isochrones', + params=isochrone_params, + headers={'Authorization': api_key}, + timeout=30 + ) + else: + # can't call isochrones api without coverage + raise HTTPInternalServerError( + 'Coverage not found for source') + + # Vérification du statut de la réponse + if response.status_code == 401: + raise HTTPInternalServerError( + 'Authentication error with Navitia API') + elif response.status_code == 400: + raise HTTPBadRequest('Invalid parameters for Navitia API') + elif response.status_code == 404: + return {} + elif not response.ok: + raise HTTPInternalServerError(f'Navitia API error: {response.status_code}') # noqa: E501 + + # Retour des données JSON + return response.json() + + except requests.exceptions.Timeout: + raise HTTPInternalServerError( + 'Timeout when calling the Navitia API') + except requests.exceptions.RequestException as e: + raise HTTPInternalServerError(f'Network error: {str(e)}') + except Exception as e: + raise HTTPInternalServerError(f'Internal error: {str(e)}') + + +def is_wp_in_isochrone(waypoint, isochrone_geom): + # get lon & lat + geom = shape(json.loads(waypoint.get("geometry").get("geom"))) + + src_epsg = 3857 + transformer = Transformer.from_crs( + f"EPSG:{src_epsg}", "EPSG:4326", always_xy=True) + lon, lat = transformer.transform(geom.x, geom.y) + pt = Point(lon, lat) + + return isochrone_geom.contains(pt) + \ No newline at end of file diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index afae3ecd1..7f59f882d 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -610,7 +610,7 @@ def worst_rating(rating1, rating2): def build_reachable_route_query(params, meta_params): """build the query based on params and meta params. this includes every filters on route, as well as offset + limit, sort, bbox... - returns a list of routes reachable (accessible by common transports), filtered with params + returns a list of routes reachable (can be accessible by public transports), filtered with params """ search = build_query(params, meta_params, ROUTE_TYPE) search_dict = search.to_dict() @@ -789,3 +789,193 @@ def build_reachable_route_query(params, meta_params): distinct() return query + + +def build_reachable_route_query_with_waypoints(params, meta_params): + """build the query based on params and meta params. + this includes every filters on route, as well as offset + limit, sort, bbox... + returns a list of routes reachable (accessible by common transports), filtered with params + """ + search = build_query(params, meta_params, ROUTE_TYPE) + search_dict = search.to_dict() + filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) + + filter_conditions = [] + + # Mapping filter keys to models + filter_map = { + 'areas': Area, + 'waypoints': Waypoint + } + + # the array of conditions to filter the query results + filter_conditions = [] + + # the array of langs available + lang = [] + + # loop over each filter + for f in filters: + for filter_key, param in f.items(): + for param_key, param_value in param.items(): + # special cases + if param_key == 'available_locales': # lang is determined by the available locales for the document + if isinstance(param_value, list): + lang = param_value + else: + lang = [param_value] + elif param_key == 'geom': + col = getattr(DocumentGeometry, 'geom') + polygon = Polygon([ + (param_value['left'], param_value['bottom']), + (param_value['right'], param_value['bottom']), + (param_value['right'], param_value['top']), + (param_value['left'], param_value['top']), + (param_value['left'], param_value['bottom']) + ]) + polygon_wkb = from_shape(polygon, srid=4326) + + filter_conditions.append(ST_Intersects( + ST_Transform(col, 4326), polygon_wkb)) + elif param_key in filter_map: # param_key is 'area' or 'waypoint' + col = getattr(filter_map[param_key], 'document_id') + if isinstance(param_value, list): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + else: # all filters on Route + # col <=> Route.param_key + col = getattr(Route, param_key) + column = col.property.columns[0] + col_type = column.type + + if filter_key == 'range': + # lte and gte are integers + gte = param_value.get('gte') + lte = param_value.get('lte') + mapper = sortable_search_attr_by_field[param_key] + values = [] + if gte is not None and lte is not None: + if gte == lte: + values = [ + val for val in mapper if mapper[val] == gte and mapper[val] == lte] + else: + # find array of possible values (not integers but enum values) between gte and lte + values = [ + val for val in mapper if mapper[val] >= gte and mapper[val] < lte] + + elif gte is not None: + # find array of possible values (not integers but enum values) >= gte + values = [ + val for val in mapper if mapper[val] >= gte] + + elif lte is not None: + # find array of possible values (not integers but enum values) < lte + values = [ + val for val in mapper if mapper[val] < lte] + + # then compare (==) col with each value + # combine multiple checks with | + checks = [(col == val) for val in values] + if len(checks) > 0: + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + + filter_conditions.append(or_expr) + + elif filter_key == 'terms': + values = param_value if isinstance( + param_value, (list, tuple)) else [param_value] + + if isinstance(col_type, ArrayOfEnum): + # combine multiple checks with | + checks = [col.any(v) for v in values] + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + filter_conditions.append(or_expr) + else: + filter_conditions.append(col.in_(values)) + + elif filter_key == 'term': + if isinstance(col_type, ArrayOfEnum): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + + else: + continue + + # combine all conditions with & + if len(filter_conditions) == 0: + filter_conditions = True + elif len(filter_conditions) == 1: + filter_conditions = filter_conditions[0] + else: + final_expr = filter_conditions[0] + for cond in filter_conditions[1:]: + final_expr = final_expr & cond + filter_conditions = final_expr + + # get sort information + sorts = search_dict.get('sort', []) + # compute sort expressions + sort_expressions = [] + for sort in sorts: + if (sort == 'undefined'): + pass + # sort by desc + elif (hasattr(sort, 'items')): + for attribute, order in sort.items(): + if (attribute == 'id'): + sort_expressions.append( + nullslast(getattr(Route, 'document_id').desc())) + else: + sort_expressions.append( + nullslast(getattr(Route, attribute).desc())) + else: + # sort by asc + sort_expressions.append(nullslast(getattr(Route, sort).asc())) + + # perform query + query = DBSession.query(Route, + func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column( + "'document_id'"), Area.document_id + ))).label("areas"), + func.jsonb_agg(func.distinct( + func.jsonb_build_object( + literal_column("'document_id'"), Waypoint.document_id + ))).label("waypoints")). \ + select_from(Association). \ + join(Route, or_( + Route.document_id == Association.child_document_id, + Route.document_id == Association.parent_document_id + )). \ + join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ + join(Waypoint, or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + )). \ + filter(Waypoint.waypoint_type == 'access'). \ + join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ + join(AreaAssociation, or_( + AreaAssociation.document_id == Association.child_document_id, + AreaAssociation.document_id == Association.parent_document_id + )). \ + join(Area, Area.document_id == AreaAssociation.area_id) + + if (len(lang) > 0): + query = query.join(DocumentLocale, and_( + DocumentLocale.document_id == Route.document_id, + DocumentLocale.lang.in_(lang) + )) + query = query. \ + filter(filter_conditions). \ + order_by(*sort_expressions). \ + group_by(Route). \ + distinct() + + return query diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index d8e212c37..8a56b6e29 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -1,4 +1,5 @@ import functools +import logging from c2corg_api.models import DBSession from c2corg_api.models.association import Association @@ -59,6 +60,8 @@ from c2corg_api.models.common.sortable_search_attributes import sortable_search_attr_by_field from sqlalchemy import nullslast +log = logging.getLogger(__name__) + # the number of routes that are included for waypoints NUM_ROUTES = 400 @@ -552,9 +555,9 @@ def update_linked_routes_public_transportation_rating(waypoint, update_types): def build_reachable_waypoints_query(params, meta_params): """build the query based on params and meta params. this includes every filters on waypoints, as well as offset + limit, sort, bbox... - returns a list of waypoints reachable (accessible by common transports), filtered with params + returns a list of waypoints reachable (can be accessible by public transports), filtered with params """ - search = build_query(params, meta_params, ROUTE_TYPE) + search = build_query(params, meta_params, WAYPOINT_TYPE) search_dict = search.to_dict() filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) @@ -610,36 +613,44 @@ def build_reachable_waypoints_query(params, meta_params): # lte and gte are integers gte = param_value.get('gte') lte = param_value.get('lte') - mapper = sortable_search_attr_by_field[param_key] - values = [] - if gte is not None and lte is not None: - if gte == lte: + mapper = sortable_search_attr_by_field.get(param_key) + if mapper is not None: + values = [] + if gte is not None and lte is not None: + if gte == lte: + values = [ + val for val in mapper if mapper[val] == gte and mapper[val] == lte] + else: + # find array of possible values (not integers but enum values) between gte and lte + values = [ + val for val in mapper if mapper[val] >= gte and mapper[val] < lte] + + elif gte is not None: + # find array of possible values (not integers but enum values) >= gte values = [ - val for val in mapper if mapper[val] == gte and mapper[val] == lte] - else: - # find array of possible values (not integers but enum values) between gte and lte + val for val in mapper if mapper[val] >= gte] + + elif lte is not None: + # find array of possible values (not integers but enum values) < lte values = [ - val for val in mapper if mapper[val] >= gte and mapper[val] < lte] - - elif gte is not None: - # find array of possible values (not integers but enum values) >= gte - values = [ - val for val in mapper if mapper[val] >= gte] - - elif lte is not None: - # find array of possible values (not integers but enum values) < lte - values = [ - val for val in mapper if mapper[val] < lte] - - # then compare (==) col with each value - # combine multiple checks with | - checks = [(col == val) for val in values] - if len(checks) > 0: - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check + val for val in mapper if mapper[val] < lte] - filter_conditions.append(or_expr) + # then compare (==) col with each value + # combine multiple checks with | + checks = [(col == val) for val in values] + if len(checks) > 0: + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + + filter_conditions.append(or_expr) + else: + if gte is not None and lte is not None: + filter_conditions.append(and_(col > gte, col < lte)) + elif gte is not None: + filter_conditions.append(col > gte) + elif lte is not None: + filter_conditions.append(col < lte) elif filter_key == 'terms': values = param_value if isinstance( @@ -716,7 +727,7 @@ def build_reachable_waypoints_query(params, meta_params): if (len(lang) > 0): query = query.join(DocumentLocale, and_( - DocumentLocale.document_id == Route.document_id, + DocumentLocale.document_id == Waypoint.document_id, DocumentLocale.lang.in_(lang) )) query = query. \ From eb86874b20ee11a9dbf7842b6659be25f8118101 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Fri, 28 Nov 2025 11:23:11 +0100 Subject: [PATCH 19/41] [fix] add q=title filter in reachable doc routes --- c2corg_api/views/route.py | 51 +++++++++++++++++++++++++++++------- c2corg_api/views/waypoint.py | 23 +++++++++++++--- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 7f59f882d..f85f19110 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -627,6 +627,23 @@ def build_reachable_route_query(params, meta_params): # the array of conditions to filter the query results filter_conditions = [] + # Manage the filter for q="words in title locales" + # extract must → multi_match → query + must_list = search_dict.get('query', {}).get('bool', {}).get('must', []) + query_value = None + + for item in must_list: + mm = item.get("multi_match") + if mm: + query_value = mm.get("query") + break + + # add LIKE filter if query exists + if query_value: + col1 = getattr(DocumentLocale, "title") + col2 = getattr(RouteLocale, "title_prefix") + filter_conditions.append(or_(col1.ilike(f"%{query_value}%"), col2.ilike(f"%{query_value}%"))) + # the array of langs available lang = [] @@ -765,6 +782,8 @@ def build_reachable_route_query(params, meta_params): Route.document_id == Association.parent_document_id )). \ join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ + join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ + join(RouteLocale, RouteLocale.id == DocumentLocale.id). \ join(Waypoint, or_( Waypoint.document_id == Association.child_document_id, Waypoint.document_id == Association.parent_document_id @@ -778,10 +797,7 @@ def build_reachable_route_query(params, meta_params): join(Area, Area.document_id == AreaAssociation.area_id) if (len(lang) > 0): - query = query.join(DocumentLocale, and_( - DocumentLocale.document_id == Route.document_id, - DocumentLocale.lang.in_(lang) - )) + query = query.filter(DocumentLocale.lang.in_(lang)) query = query. \ filter(filter_conditions). \ order_by(*sort_expressions). \ @@ -810,6 +826,23 @@ def build_reachable_route_query_with_waypoints(params, meta_params): # the array of conditions to filter the query results filter_conditions = [] + + # Manage the filter for q="words in title locales" + # extract must → multi_match → query + must_list = search_dict.get('query', {}).get('bool', {}).get('must', []) + query_value = None + + for item in must_list: + mm = item.get("multi_match") + if mm: + query_value = mm.get("query") + break + + # add LIKE filter if query exists + if query_value: + col1 = getattr(DocumentLocale, "title") + col2 = getattr(RouteLocale, "title_prefix") + filter_conditions.append(or_(col1.ilike(f"%{query_value}%"), col2.ilike(f"%{query_value}%"))) # the array of langs available lang = [] @@ -947,7 +980,8 @@ def build_reachable_route_query_with_waypoints(params, meta_params): ))).label("areas"), func.jsonb_agg(func.distinct( func.jsonb_build_object( - literal_column("'document_id'"), Waypoint.document_id + literal_column( + "'document_id'"), Waypoint.document_id ))).label("waypoints")). \ select_from(Association). \ join(Route, or_( @@ -955,6 +989,8 @@ def build_reachable_route_query_with_waypoints(params, meta_params): Route.document_id == Association.parent_document_id )). \ join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ + join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ + join(RouteLocale, RouteLocale.id == DocumentLocale.id). \ join(Waypoint, or_( Waypoint.document_id == Association.child_document_id, Waypoint.document_id == Association.parent_document_id @@ -968,10 +1004,7 @@ def build_reachable_route_query_with_waypoints(params, meta_params): join(Area, Area.document_id == AreaAssociation.area_id) if (len(lang) > 0): - query = query.join(DocumentLocale, and_( - DocumentLocale.document_id == Route.document_id, - DocumentLocale.lang.in_(lang) - )) + query = query.filter(DocumentLocale.lang.in_(lang)) query = query. \ filter(filter_conditions). \ order_by(*sort_expressions). \ diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index 8a56b6e29..33aa31a92 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -570,6 +570,22 @@ def build_reachable_waypoints_query(params, meta_params): # the array of conditions to filter the query results filter_conditions = [] + + # Manage the filter for q="words in title locales" + # extract must → multi_match → query + must_list = search_dict.get('query', {}).get('bool', {}).get('must', []) + query_value = None + + for item in must_list: + mm = item.get("multi_match") + if mm: + query_value = mm.get("query") + break + + # add LIKE filter if query exists + if query_value: + col = getattr(DocumentLocale, "title") + filter_conditions.append(col.ilike(f"%{query_value}%")) # the array of langs available lang = [] @@ -717,6 +733,7 @@ def build_reachable_waypoints_query(params, meta_params): Waypoint.document_id == Association.parent_document_id )). \ join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ + join(DocumentLocale, Waypoint.document_id == DocumentLocale.document_id). \ filter(Waypoint.waypoint_type == 'access'). \ join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ join(AreaAssociation, or_( @@ -726,10 +743,8 @@ def build_reachable_waypoints_query(params, meta_params): join(Area, Area.document_id == AreaAssociation.area_id) if (len(lang) > 0): - query = query.join(DocumentLocale, and_( - DocumentLocale.document_id == Waypoint.document_id, - DocumentLocale.lang.in_(lang) - )) + query = query.filter(DocumentLocale.lang.in_(lang)) + query = query. \ filter(filter_conditions). \ order_by(*sort_expressions). \ From cfa6f73c54233beafc0597c71f348461bb08fd12 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Fri, 28 Nov 2025 16:24:35 +0100 Subject: [PATCH 20/41] [fix] make sure waypoints are not reachable if journeys departure day is not the same as the day in parameters --- c2corg_api/views/navitia.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index 16a03ece0..fa9e82f8b 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -501,9 +501,14 @@ def is_wp_journey_reachable(waypoint, journey_params): elif not response.ok: return False - # Retour des données JSON - #TODO : return au moins 1 journey pour qui le jour de la date de départ (departure_date_time) est le même que le jour dans journey_params.date_time - return True + # make sure the waypoint is reachable if at least one journey's departure date time is the same day as the day in journey_params + for journey in response.json()['journeys']: + journey_day = int(journey['departure_date_time'][6:8]) + param_day = int(journey_params['datetime'][6:8]) + if journey_day == param_day: + return True + + return False except requests.exceptions.Timeout: return False From 5f13610bbd217c75e5cbee94a8da93ddb8ffbdba Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 1 Dec 2025 13:54:48 +0100 Subject: [PATCH 21/41] [refactor] organization + documentation : factorization and helper functions for Itinevert routes --- c2corg_api/search/utils.py | 253 +++++++++++++++++++++++ c2corg_api/views/navitia.py | 334 ++++++++++++++++-------------- c2corg_api/views/route.py | 379 +++++------------------------------ c2corg_api/views/waypoint.py | 199 +++--------------- 4 files changed, 516 insertions(+), 649 deletions(-) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index e06b27a6c..f42d3b86f 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -1,4 +1,13 @@ +import logging +from operator import and_, or_ import re +from shapely.geometry import Polygon +from geoalchemy2.shape import from_shape +from sqlalchemy import nullslast +from geoalchemy2.functions import ST_Intersects, ST_Transform +from c2corg_api.models.utils import ArrayOfEnum + +log = logging.getLogger(__name__) BBCODE_TAGS = [ 'b', 'i', 'u', 's', 'q', 'c', 'sup', 'ind', 'url', 'email', 'acr(onym)?', @@ -31,3 +40,247 @@ def strip_bbcodes(s): def get_title(title, title_prefix): return title_prefix + ' : ' + title if title_prefix else title + + +def build_sqlalchemy_filters( + search_dict, # elastic search dict + document_model, # the model (waypoint, routes, etc...) + filter_map, # for multicriteria search (ex : searching a waypoint by area id) + geometry_model, # the Geometry model (where ce access to geometry), most likely always DocumentGeometry + range_enum_map, # the mapper for range enum, most likely always sortable_search_attr_by_field + title_columns=None # the column for the title (ex: Waypoint -> title, Route -> title and title_prefix) +): + """ + Build SQLAlchemy filter for documents (Waypoint, Route, etc.) based on filters that would normally be used by ElasticSearch + + this can then be used to filter directly in a DB query + + Usage Example : + + search = build_query(params, meta_params, WAYPOINT_TYPE) + + search_dict = search.to_dict() + + filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( + search_dict=search_dict, + document_model=Waypoint, + filter_map={"areas": Area,}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title] + ) + + query = DBSession.query(Waypoint).filter(filter_conditions).order_by(*sort_expressions) + + """ + + filters = search_dict.get("query", {}).get("bool", {}).get("filter", []) + must_list = search_dict.get("query", {}).get("bool", {}).get("must", []) + + filter_conditions = [] + needs_locale_join = False + langs = [] + + # corresponds to the elastic search ?q= which looks for title + # use title_columns to specify in which columns to look for + query_value = None + for item in must_list: + mm = item.get("multi_match") + if mm: + query_value = mm.get("query") + break + + if query_value and title_columns: + needs_locale_join = True + like_clauses = [col.ilike(f"%{query_value}%") for col in title_columns] + filter_conditions.append(or_(*like_clauses)) + + # loop over all elastic search filters + for f in filters: + for filter_key, param in f.items(): + + for param_key, param_value in param.items(): + + # available_locales to get langs + if param_key == "available_locales": + langs = param_value if isinstance( + param_value, list) else [param_value] + + # geometry-based filtering -> bbox + elif param_key == "geom": + col = getattr(geometry_model, "geom") + polygon = Polygon([ + (param_value["left"], param_value["bottom"]), + (param_value["right"], param_value["bottom"]), + (param_value["right"], param_value["top"]), + (param_value["left"], param_value["top"]), + (param_value["left"], param_value["bottom"]), + ]) + polygon_wkb = from_shape(polygon, srid=4326) + filter_conditions.append(ST_Intersects( + ST_Transform(col, 4326), polygon_wkb)) + + # special cases of documents associated to other doc + elif param_key in filter_map: + col = getattr(filter_map[param_key], "document_id") + if isinstance(param_value, list): + filter_conditions.append(col.any(param_value)) + else: + filter_conditions.append(col == param_value) + + # generic attribute filters on the document model + else: + col = getattr(document_model, param_key) + column = col.property.columns[0] + col_type = column.type + + # for range attributes + if filter_key == "range": + filter_conditions.append( + build_range_expression( + col, param_value, range_enum_map.get(param_key)) + ) + + # for terms + elif filter_key == "terms": + values = param_value if isinstance( + param_value, (list, tuple)) else [param_value] + filter_conditions.append( + build_terms_expression(col, values, col_type) + ) + + # for term + elif filter_key == "term": + filter_conditions.append( + build_term_expression(col, param_value, col_type) + ) + + else: + continue + + # combine and conditions + final_filter = combine_conditions(filter_conditions) + + log.warning(final_filter) + + # build sort expressions + sort_expressions = build_sort_expressions( + search_dict.get("sort", []), document_model + ) + + # return each valuable variable to be used later in a sql alchemy DBSession.query + return final_filter, sort_expressions, needs_locale_join, langs + + +def build_range_expression(col, param_value, enum_map): + """ + build sql alchemy filter for range expressions + """ + gte = param_value.get("gte") + lte = param_value.get("lte") + + # ENUM RANGE (enum_map: value -> number) + if enum_map: + values = [] + if gte is not None and lte is not None: + if gte == lte: + values = [val for val, num in enum_map.items() if num == gte] + else: + values = [val for val, num in enum_map.items() if num >= gte and num < lte] + elif gte is not None: + values = [val for val, num in enum_map.items() if num >= gte] + elif lte is not None: + values = [val for val, num in enum_map.items() if num < lte] + + checks = [col == v for v in values] + if not checks: + return False + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + return or_expr + + # NUMERIC RANGE + clauses = [] + if gte is not None: + clauses.append(col >= gte) + if lte is not None: + clauses.append(col <= lte) + if not clauses: + return False + return and_(*clauses) + +def build_terms_expression(col, values, col_type): + """ + build sql alchemy filter for terms expressions + """ + # normalize values to list/tuple + values = values if isinstance(values, (list, tuple)) else [values] + if not values: + return True + + if isinstance(col_type, ArrayOfEnum): + checks = [col.any(v) for v in values] + if not checks: + return True + # build OR by folding with | + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + return or_expr + + # non-array enum + if len(values) == 1: + return col == values[0] + return col.in_(values) + +def build_term_expression(col, value, col_type): + """ + build sql alchemy filter for term expressions + """ + if isinstance(col_type, ArrayOfEnum): + return col.any(value) + return col == value + + +def combine_conditions(conditions): + """ + useful functions to combine conditions to later apply them in a .filter + """ + if not conditions: + return True + if len(conditions) == 1: + return conditions[0] + expr = conditions[0] + for c in conditions[1:]: + expr = expr & c + return expr + + +def build_sort_expressions(sort_config, document_model): + """ + build sql alchemy sort expressions + """ + sort_expressions = [] + + for sort in sort_config: + if sort == "undefined": + continue + + # DESC + if hasattr(sort, "items"): + for attr, order in sort.items(): + col = ( + getattr(document_model, "document_id") + if attr == "id" else getattr(document_model, attr) + ) + sort_expressions.append( + nullslast(col.desc() if order == "desc" else col.asc()) + ) + + # ASC + else: + col = getattr(document_model, sort) + sort_expressions.append(nullslast(col.asc())) + + return sort_expressions diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index fa9e82f8b..2fe0a567d 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -23,6 +23,11 @@ log = logging.getLogger(__name__) +# When editing these constants, make sure to edit them in the front too (itinevert-service) +MAX_ROUTE_THRESHOLD = 50 +MAX_TRIP_DURATION = 240 +MIN_TRIP_DURATION = 20 + def validate_navitia_params(request, **kwargs): """Validates the required parameters for the Navitia API""" required_params = ['from', 'to', 'datetime', 'datetime_represents'] @@ -125,29 +130,31 @@ def get(self): raise HTTPInternalServerError(f'Internal error: {str(e)}') +def validate_journey_reachable_params(request, **kwargs): + """Validates the required parameters for the journey reachable doc route""" + required_params = ['from', 'datetime', 'datetime_represents', 'walking_speed', 'max_walking_duration_to_pt'] + + for param in required_params: + if param not in request.params: + request.errors.add( + 'querystring', + param, + f'Paramètre {param} requis') + @resource(path='/navitia/journeyreachableroutes', cors_policy=cors_policy) class NavitiaJourneyReachableRoutesRest: def __init__(self, request, context=None): self.request = request - @view(validators=[]) + @view(validators=[validate_journey_reachable_params]) def get(self): - validated = self.request.validated - - meta_params = { - 'offset': validated.get('offset', 0), - 'limit': validated.get('limit', LIMIT_DEFAULT), - 'lang': validated.get('lang') - } - - journey_params = { - 'from': self.request.params.get('from'), - 'datetime': self.request.params.get('datetime'), - 'datetime_represents': self.request.params.get('datetime_represents'), - 'walking_speed': self.request.params.get('walking_speed'), - 'max_walking_duration_to_pt': self.request.params.get('max_walking_duration_to_pt'), - 'to': '' - } + """ + Get all routes matching filters in params, that are reachable (means there exists a Navitia journey for at least one of their waypoints of type access). + NOTE : the number of routes after applying filters, has to be < MAX_ROUTE_THRESHOLD, to reduce number of queries towards Navitia journey API + """ + meta_params = extract_meta_params(self.request) + + journey_params = extract_journey_params(self.request) query = build_reachable_route_query_with_waypoints( self.request.GET, meta_params) @@ -155,37 +162,18 @@ def get(self): results = ( query.all() ) + + # /!\ IMPORTANT + # Before doing any further computations, make sure the number of routes don't go above threshold. + # The Itinevert UI doesn't allow that, but a user could query the api with wrong parameters... + if len(results) > MAX_ROUTE_THRESHOLD: + raise HTTPBadRequest("Couldn't proceed with the computation : Too much routes found.") - # manage areas for routes - areas_id = set() - for route, areas, waypoints in results: - if areas is None: - continue - for area in areas: - area_id = area.get("document_id") - if area_id is not None: - areas_id.add(area_id) - - areas_objects = DBSession.query(Area).filter( - Area.document_id.in_(areas_id)).all() - - areas_map = {area.document_id: area for area in areas_objects} - - # manage waypoints - waypoints_id = set() - for route, areas, waypoints in results: - if waypoints is None: - return - - for waypoint in waypoints: - wp_id = waypoint.get("document_id") - if wp_id is not None: - waypoints_id.add(wp_id) + areas_map = collect_areas_from_results(results, 1) - wp_objects = DBSession.query(Waypoint).filter( - Waypoint.document_id.in_(waypoints_id)).all() + wp_objects = collect_waypoints_from_results(results) - log.warning("Number of NAVITIA journey queries : %d", len(wp_objects)) + log.info("Number of NAVITIA journey queries : %d", len(wp_objects)) navitia_wp_map = {wp.document_id: is_wp_journey_reachable( to_json_dict(wp, schema_waypoint), journey_params) for wp in wp_objects} @@ -216,30 +204,33 @@ def get(self): return {'documents': routes, 'total': len(routes)} - @resource(path='/navitia/journeyreachablewaypoints', cors_policy=cors_policy) class NavitiaJourneyReachableWaypointsRest: def __init__(self, request, context=None): self.request = request - @view(validators=[]) + @view(validators=[validate_journey_reachable_params]) def get(self): - validated = self.request.validated - - meta_params = { - 'offset': validated.get('offset', 0), - 'limit': validated.get('limit', LIMIT_DEFAULT), - 'lang': validated.get('lang') - } - - journey_params = { - 'from': self.request.params.get('from'), - 'datetime': self.request.params.get('datetime'), - 'datetime_represents': self.request.params.get('datetime_represents'), - 'walking_speed': self.request.params.get('walking_speed'), - 'max_walking_duration_to_pt': self.request.params.get('max_walking_duration_to_pt'), - 'to': '' - } + """ + Get all waypoints matching filters in params, that are reachable (means there exists a Navitia journey). + NOTE : waypoints should be filtered with one area, to reduce the number of queries towards Navitia journey API. + """ + meta_params = extract_meta_params(self.request) + + journey_params = extract_journey_params(self.request) + + areas = None + try: + areas = self.request.GET['a'].split(",") + except Exception as e: + areas = None + + # Normalize: allow single value or list + if areas is None: + raise HTTPBadRequest('Missing filter : area is required') + elif isinstance(areas, list): + if len(areas) > 1: + raise HTTPBadRequest('Only one filtering area is allowed') query = build_reachable_waypoints_query( self.request.GET, meta_params) @@ -248,22 +239,9 @@ def get(self): query.all() ) - # manage areas for waypoints - areas_id = set() - for waypoints, areas in results: - if areas is None: - continue - for area in areas: - area_id = area.get("document_id") - if area_id is not None: - areas_id.add(area_id) - - areas_objects = DBSession.query(Area).filter( - Area.document_id.in_(areas_id)).all() + areas_map = collect_areas_from_results(results, 1) - areas_map = {area.document_id: area for area in areas_objects} - - log.warning("Number of NAVITIA journey queries : %d", len(results)) + log.info("Number of NAVITIA journey queries : %d", len(results)) waypoints = [] for waypoint, areas in results: @@ -286,47 +264,38 @@ def get(self): return {'documents': waypoints, 'total': len(waypoints)} +def validate_isochrone_reachable_params(request, **kwargs): + """Validates the required parameters for the isochrone reachable doc route""" + required_params = ['from', 'datetime', 'datetime_represents', 'boundary_duration'] + + for param in required_params: + if param not in request.params: + request.errors.add( + 'querystring', + param, + f'Paramètre {param} requis') + @resource(path='/navitia/isochronesreachableroutes', cors_policy=cors_policy) class NavitiaIsochronesReachableRoutesRest: def __init__(self, request, context=None): self.request = request - @view(validators=[]) + @view(validators=[validate_isochrone_reachable_params]) def get(self): - validated = self.request.validated - - meta_params = { - 'offset': validated.get('offset', 0), - 'limit': validated.get('limit', LIMIT_DEFAULT), - 'lang': validated.get('lang') - } + """ + Get all routes matching filters in params, that have at least one waypoint of type access that is inside the isochron. + The isochron is created by querying navitia api with specific parameters, see validate_isochrone_reachable_params func + """ + meta_params = extract_meta_params(self.request) - isochrone_params = { - 'from': self.request.params.get('from'), - 'datetime': self.request.params.get('datetime'), - 'boundary_duration[]': self.request.params.get('boundary_duration'), - 'datetime_represents': self.request.params.get('datetime_represents') - } + isochrone_params = extract_isochrone_params(self.request) query = build_reachable_route_query_with_waypoints( self.request.GET, meta_params) results = query.all() - # manage areas for routes - areas_id = set() - for route, areas, waypoints in results: - if areas is None: - continue - for area in areas: - area_id = area.get("document_id") - if area_id is not None: - areas_id.add(area_id) - - areas_objects = DBSession.query(Area).filter( - Area.document_id.in_(areas_id)).all() - - areas_map = {area.document_id: area for area in areas_objects} + areas_map = collect_areas_from_results(results, 1) response = get_navitia_isochrone(isochrone_params) @@ -337,19 +306,7 @@ def get(self): geojson = response["isochrones"][0]["geojson"] isochrone_geom = shape(geojson) - # manage waypoints - waypoints_id = set() - for route, areas, waypoints in results: - if waypoints is None: - return - - for waypoint in waypoints: - wp_id = waypoint.get("document_id") - if wp_id is not None: - waypoints_id.add(wp_id) - - wp_objects = DBSession.query(Waypoint).filter( - Waypoint.document_id.in_(waypoints_id)).all() + wp_objects = collect_waypoints_from_results(results) navitia_wp_map = {wp.document_id: is_wp_in_isochrone( to_json_dict(wp, schema_waypoint), isochrone_geom) for wp in wp_objects} @@ -386,22 +343,15 @@ class NavitiaIsochronesReachableWaypointsRest: def __init__(self, request, context=None): self.request = request - @view(validators=[]) + @view(validators=[validate_isochrone_reachable_params]) def get(self): - validated = self.request.validated - - meta_params = { - 'offset': validated.get('offset', 0), - 'limit': validated.get('limit', LIMIT_DEFAULT), - 'lang': validated.get('lang') - } + """ + Get all waypoints matching filters in params, that are inside the isochron. + The isochron is created by querying navitia api with specific parameters, see validate_isochrone_reachable_params func + """ + meta_params = extract_meta_params(self.request) - isochrone_params = { - 'from': self.request.params.get('from'), - 'datetime': self.request.params.get('datetime'), - 'boundary_duration[]': self.request.params.get('boundary_duration'), - 'datetime_represents': self.request.params.get('datetime_represents') - } + isochrone_params = extract_isochrone_params(self.request) query = build_reachable_waypoints_query( self.request.GET, meta_params) @@ -409,19 +359,7 @@ def get(self): results = query.all() # manage areas for waypoints - areas_id = set() - for waypoints, areas in results: - if areas is None: - continue - for area in areas: - area_id = area.get("document_id") - if area_id is not None: - areas_id.add(area_id) - - areas_objects = DBSession.query(Area).filter( - Area.document_id.in_(areas_id)).all() - - areas_map = {area.document_id: area for area in areas_objects} + areas_map = collect_areas_from_results(results, 1) response = get_navitia_isochrone(isochrone_params) @@ -454,6 +392,10 @@ def get(self): def is_wp_journey_reachable(waypoint, journey_params): + """ + Query the navitia Journey api and returns true if the waypoint is reachable (at least one journey has been found) + NOTE : the journey's departure time has to be the same day as the datetime's day in journey_params + """ # enhance journey params with the 'to' parameter, from the waypoint geometry. geom = shape(json.loads(waypoint.get("geometry").get("geom"))) @@ -519,6 +461,9 @@ def is_wp_journey_reachable(waypoint, journey_params): def get_navitia_isochrone(isochrone_params): + """ + Query the navitia Isochrones api, and returns the isochrone object + """ lon = isochrone_params.get("from").split(";")[0] lat = isochrone_params.get("from").split(";")[1] source_coverage = get_coverage(lon, lat) @@ -569,6 +514,9 @@ def get_navitia_isochrone(isochrone_params): def is_wp_in_isochrone(waypoint, isochrone_geom): + """ + Returns true if waypoints is contained in isochrone_geom + """ # get lon & lat geom = shape(json.loads(waypoint.get("geometry").get("geom"))) @@ -579,4 +527,96 @@ def is_wp_in_isochrone(waypoint, isochrone_geom): pt = Point(lon, lat) return isochrone_geom.contains(pt) - \ No newline at end of file + + +def extract_meta_params(request): + """ + Extract meta parameters such as offset, limit and lang + """ + v = request.validated + return { + 'offset': v.get('offset', 0), + 'limit': v.get('limit', LIMIT_DEFAULT), + 'lang': v.get('lang'), + } + + +def extract_journey_params(request): + """ + Extract parameters for journey query + """ + return { + 'from': request.params.get('from'), + 'datetime': request.params.get('datetime'), + 'datetime_represents': request.params.get('datetime_represents'), + 'walking_speed': request.params.get('walking_speed'), + 'max_walking_duration_to_pt': request.params.get('max_walking_duration_to_pt'), + 'to': '' + } + + +def extract_isochrone_params(request): + """ + Extract parameters for isochrone query + NOTE : the boundary duration is bounded by constants MAX_TRIP_DURATION and MIN_TRIP_DURATION + if the boundary duration goes beyond limits, it is set to the limit it goes past. + """ + params = { + 'from': request.params.get('from'), + 'datetime': request.params.get('datetime'), + 'boundary_duration[]': request.params.get('boundary_duration'), + 'datetime_represents': request.params.get('datetime_represents') + } + # normalize boundary + bd = params['boundary_duration[]'] + if len(bd.split(",")) == 1: + duration = int(bd) + params['boundary_duration[]'] = max(min(duration, MAX_TRIP_DURATION * 60), + MIN_TRIP_DURATION * 60) + return params + +def collect_areas_from_results(results, areaIndex): + """ + Extract all area document_ids from results, load Area objects from DB, + and return {document_id: Area}. + """ + area_ids = set() + + for row in results: + areas = row[areaIndex] + + if not areas: + continue + + for area in areas: + doc_id = area.get("document_id") + if doc_id: + area_ids.add(doc_id) + + area_objects = DBSession.query(Area).filter( + Area.document_id.in_(area_ids) + ).all() + + return {a.document_id: a for a in area_objects} + +def collect_waypoints_from_results(results): + """ + Extract all waypoint document_ids from results, load Waypoint objects from DB, + and return {document_id: Waypoint}. + """ + wp_ids = set() + + for route, areas, waypoints in results: + if not waypoints: + continue + + for wp in waypoints: + doc_id = wp.get("document_id") + if doc_id: + wp_ids.add(doc_id) + + wp_objects = DBSession.query(Waypoint).filter( + Waypoint.document_id.in_(wp_ids) + ).all() + + return {wp for wp in wp_objects} \ No newline at end of file diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index f85f19110..21c22cb9b 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -7,6 +7,7 @@ from c2corg_api.models.document import DocumentLocale, DocumentGeometry from c2corg_api.models.outing import Outing from c2corg_api.models.waypoint import WAYPOINT_TYPE, Waypoint +from c2corg_api.search.utils import build_sqlalchemy_filters from c2corg_api.security.acl import ACLDefault from c2corg_api.views.document_associations import get_first_column from c2corg_api.views.document_info import DocumentInfoRest @@ -614,162 +615,15 @@ def build_reachable_route_query(params, meta_params): """ search = build_query(params, meta_params, ROUTE_TYPE) search_dict = search.to_dict() - filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) - - filter_conditions = [] - - # Mapping filter keys to models - filter_map = { - 'areas': Area, - 'waypoints': Waypoint - } - - # the array of conditions to filter the query results - filter_conditions = [] - - # Manage the filter for q="words in title locales" - # extract must → multi_match → query - must_list = search_dict.get('query', {}).get('bool', {}).get('must', []) - query_value = None - - for item in must_list: - mm = item.get("multi_match") - if mm: - query_value = mm.get("query") - break - - # add LIKE filter if query exists - if query_value: - col1 = getattr(DocumentLocale, "title") - col2 = getattr(RouteLocale, "title_prefix") - filter_conditions.append(or_(col1.ilike(f"%{query_value}%"), col2.ilike(f"%{query_value}%"))) - - # the array of langs available - lang = [] - - # loop over each filter - for f in filters: - for filter_key, param in f.items(): - for param_key, param_value in param.items(): - # special cases - if param_key == 'available_locales': # lang is determined by the available locales for the document - if isinstance(param_value, list): - lang = param_value - else: - lang = [param_value] - elif param_key == 'geom': - col = getattr(DocumentGeometry, 'geom') - polygon = Polygon([ - (param_value['left'], param_value['bottom']), - (param_value['right'], param_value['bottom']), - (param_value['right'], param_value['top']), - (param_value['left'], param_value['top']), - (param_value['left'], param_value['bottom']) - ]) - polygon_wkb = from_shape(polygon, srid=4326) - - filter_conditions.append(ST_Intersects( - ST_Transform(col, 4326), polygon_wkb)) - elif param_key in filter_map: # param_key is 'area' or 'waypoint' - col = getattr(filter_map[param_key], 'document_id') - if isinstance(param_value, list): - filter_conditions.append(col.any(param_value)) - else: - filter_conditions.append(col == param_value) - else: # all filters on Route - # col <=> Route.param_key - col = getattr(Route, param_key) - column = col.property.columns[0] - col_type = column.type - - if filter_key == 'range': - # lte and gte are integers - gte = param_value.get('gte') - lte = param_value.get('lte') - mapper = sortable_search_attr_by_field[param_key] - values = [] - if gte is not None and lte is not None: - if gte == lte: - values = [ - val for val in mapper if mapper[val] == gte and mapper[val] == lte] - else: - # find array of possible values (not integers but enum values) between gte and lte - values = [ - val for val in mapper if mapper[val] >= gte and mapper[val] < lte] - - elif gte is not None: - # find array of possible values (not integers but enum values) >= gte - values = [ - val for val in mapper if mapper[val] >= gte] - - elif lte is not None: - # find array of possible values (not integers but enum values) < lte - values = [ - val for val in mapper if mapper[val] < lte] - - # then compare (==) col with each value - # combine multiple checks with | - checks = [(col == val) for val in values] - if len(checks) > 0: - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check - - filter_conditions.append(or_expr) - - elif filter_key == 'terms': - values = param_value if isinstance( - param_value, (list, tuple)) else [param_value] - - if isinstance(col_type, ArrayOfEnum): - # combine multiple checks with | - checks = [col.any(v) for v in values] - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check - filter_conditions.append(or_expr) - else: - filter_conditions.append(col.in_(values)) - - elif filter_key == 'term': - if isinstance(col_type, ArrayOfEnum): - filter_conditions.append(col.any(param_value)) - else: - filter_conditions.append(col == param_value) - - else: - continue - - # combine all conditions with & - if len(filter_conditions) == 0: - filter_conditions = True - elif len(filter_conditions) == 1: - filter_conditions = filter_conditions[0] - else: - final_expr = filter_conditions[0] - for cond in filter_conditions[1:]: - final_expr = final_expr & cond - filter_conditions = final_expr - - # get sort information - sorts = search_dict.get('sort', []) - # compute sort expressions - sort_expressions = [] - for sort in sorts: - if (sort == 'undefined'): - pass - # sort by desc - elif (hasattr(sort, 'items')): - for attribute, order in sort.items(): - if (attribute == 'id'): - sort_expressions.append( - nullslast(getattr(Route, 'document_id').desc())) - else: - sort_expressions.append( - nullslast(getattr(Route, attribute).desc())) - else: - # sort by asc - sort_expressions.append(nullslast(getattr(Route, sort).asc())) + + filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( + search_dict, + document_model=Route, + filter_map={'areas': Area, 'waypoints': Waypoint}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title, RouteLocale.title_prefix] + ) # perform query query = DBSession.query(Route, func.jsonb_agg(func.distinct( @@ -781,23 +635,29 @@ def build_reachable_route_query(params, meta_params): Route.document_id == Association.child_document_id, Route.document_id == Association.parent_document_id )). \ - join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ - join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ - join(RouteLocale, RouteLocale.id == DocumentLocale.id). \ - join(Waypoint, or_( - Waypoint.document_id == Association.child_document_id, - Waypoint.document_id == Association.parent_document_id + join(Waypoint, and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' )). \ - filter(Waypoint.waypoint_type == 'access'). \ join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ + join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id )). \ join(Area, Area.document_id == AreaAssociation.area_id) - if (len(lang) > 0): - query = query.filter(DocumentLocale.lang.in_(lang)) + if (needs_locale_join): + query = query. \ + join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ + join(RouteLocale, RouteLocale.id == DocumentLocale.id) + + if (len(langs) > 0): + query = query.filter(DocumentLocale.lang.in_(langs)) + query = query. \ filter(filter_conditions). \ order_by(*sort_expressions). \ @@ -814,163 +674,16 @@ def build_reachable_route_query_with_waypoints(params, meta_params): """ search = build_query(params, meta_params, ROUTE_TYPE) search_dict = search.to_dict() - filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) - - filter_conditions = [] - - # Mapping filter keys to models - filter_map = { - 'areas': Area, - 'waypoints': Waypoint - } - - # the array of conditions to filter the query results - filter_conditions = [] - # Manage the filter for q="words in title locales" - # extract must → multi_match → query - must_list = search_dict.get('query', {}).get('bool', {}).get('must', []) - query_value = None - - for item in must_list: - mm = item.get("multi_match") - if mm: - query_value = mm.get("query") - break - - # add LIKE filter if query exists - if query_value: - col1 = getattr(DocumentLocale, "title") - col2 = getattr(RouteLocale, "title_prefix") - filter_conditions.append(or_(col1.ilike(f"%{query_value}%"), col2.ilike(f"%{query_value}%"))) - - # the array of langs available - lang = [] - - # loop over each filter - for f in filters: - for filter_key, param in f.items(): - for param_key, param_value in param.items(): - # special cases - if param_key == 'available_locales': # lang is determined by the available locales for the document - if isinstance(param_value, list): - lang = param_value - else: - lang = [param_value] - elif param_key == 'geom': - col = getattr(DocumentGeometry, 'geom') - polygon = Polygon([ - (param_value['left'], param_value['bottom']), - (param_value['right'], param_value['bottom']), - (param_value['right'], param_value['top']), - (param_value['left'], param_value['top']), - (param_value['left'], param_value['bottom']) - ]) - polygon_wkb = from_shape(polygon, srid=4326) - - filter_conditions.append(ST_Intersects( - ST_Transform(col, 4326), polygon_wkb)) - elif param_key in filter_map: # param_key is 'area' or 'waypoint' - col = getattr(filter_map[param_key], 'document_id') - if isinstance(param_value, list): - filter_conditions.append(col.any(param_value)) - else: - filter_conditions.append(col == param_value) - else: # all filters on Route - # col <=> Route.param_key - col = getattr(Route, param_key) - column = col.property.columns[0] - col_type = column.type - - if filter_key == 'range': - # lte and gte are integers - gte = param_value.get('gte') - lte = param_value.get('lte') - mapper = sortable_search_attr_by_field[param_key] - values = [] - if gte is not None and lte is not None: - if gte == lte: - values = [ - val for val in mapper if mapper[val] == gte and mapper[val] == lte] - else: - # find array of possible values (not integers but enum values) between gte and lte - values = [ - val for val in mapper if mapper[val] >= gte and mapper[val] < lte] - - elif gte is not None: - # find array of possible values (not integers but enum values) >= gte - values = [ - val for val in mapper if mapper[val] >= gte] - - elif lte is not None: - # find array of possible values (not integers but enum values) < lte - values = [ - val for val in mapper if mapper[val] < lte] - - # then compare (==) col with each value - # combine multiple checks with | - checks = [(col == val) for val in values] - if len(checks) > 0: - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check - - filter_conditions.append(or_expr) - - elif filter_key == 'terms': - values = param_value if isinstance( - param_value, (list, tuple)) else [param_value] - - if isinstance(col_type, ArrayOfEnum): - # combine multiple checks with | - checks = [col.any(v) for v in values] - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check - filter_conditions.append(or_expr) - else: - filter_conditions.append(col.in_(values)) - - elif filter_key == 'term': - if isinstance(col_type, ArrayOfEnum): - filter_conditions.append(col.any(param_value)) - else: - filter_conditions.append(col == param_value) - - else: - continue - - # combine all conditions with & - if len(filter_conditions) == 0: - filter_conditions = True - elif len(filter_conditions) == 1: - filter_conditions = filter_conditions[0] - else: - final_expr = filter_conditions[0] - for cond in filter_conditions[1:]: - final_expr = final_expr & cond - filter_conditions = final_expr - - # get sort information - sorts = search_dict.get('sort', []) - # compute sort expressions - sort_expressions = [] - for sort in sorts: - if (sort == 'undefined'): - pass - # sort by desc - elif (hasattr(sort, 'items')): - for attribute, order in sort.items(): - if (attribute == 'id'): - sort_expressions.append( - nullslast(getattr(Route, 'document_id').desc())) - else: - sort_expressions.append( - nullslast(getattr(Route, attribute).desc())) - else: - # sort by asc - sort_expressions.append(nullslast(getattr(Route, sort).asc())) - + filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( + search_dict, + document_model=Route, + filter_map={'areas': Area, 'waypoints': Waypoint}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title, RouteLocale.title_prefix] + ) + # perform query query = DBSession.query(Route, func.jsonb_agg(func.distinct( @@ -988,23 +701,29 @@ def build_reachable_route_query_with_waypoints(params, meta_params): Route.document_id == Association.child_document_id, Route.document_id == Association.parent_document_id )). \ - join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ - join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ - join(RouteLocale, RouteLocale.id == DocumentLocale.id). \ - join(Waypoint, or_( - Waypoint.document_id == Association.child_document_id, - Waypoint.document_id == Association.parent_document_id + join(Waypoint, and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' )). \ - filter(Waypoint.waypoint_type == 'access'). \ join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ + join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id )). \ join(Area, Area.document_id == AreaAssociation.area_id) - if (len(lang) > 0): - query = query.filter(DocumentLocale.lang.in_(lang)) + if (needs_locale_join): + query = query. \ + join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ + join(RouteLocale, RouteLocale.id == DocumentLocale.id) + + if (len(langs) > 0): + query = query.filter(DocumentLocale.lang.in_(langs)) + query = query. \ filter(filter_conditions). \ order_by(*sort_expressions). \ diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index 33aa31a92..232ccf175 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -6,6 +6,7 @@ from c2corg_api.models.document import UpdateType from c2corg_api.models.outing import Outing from c2corg_api.models.route import Route, RouteLocale, ROUTE_TYPE +from c2corg_api.search.utils import build_sqlalchemy_filters from c2corg_api.views.document_associations import get_first_column from c2corg_api.views.document_info import DocumentInfoRest from c2corg_api.views.document_listings import get_documents_for_ids @@ -558,192 +559,46 @@ def build_reachable_waypoints_query(params, meta_params): returns a list of waypoints reachable (can be accessible by public transports), filtered with params """ search = build_query(params, meta_params, WAYPOINT_TYPE) - search_dict = search.to_dict() - filters = search_dict.get('query', {}).get('bool', {}).get('filter', []) - - filter_conditions = [] - # Mapping filter keys to models - filter_map = { - 'areas': Area - } + search_dict = search.to_dict() - # the array of conditions to filter the query results - filter_conditions = [] + filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( + search_dict=search_dict, + document_model=Waypoint, + filter_map={"areas": Area,}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title] + ) - # Manage the filter for q="words in title locales" - # extract must → multi_match → query - must_list = search_dict.get('query', {}).get('bool', {}).get('must', []) - query_value = None - - for item in must_list: - mm = item.get("multi_match") - if mm: - query_value = mm.get("query") - break - - # add LIKE filter if query exists - if query_value: - col = getattr(DocumentLocale, "title") - filter_conditions.append(col.ilike(f"%{query_value}%")) - - # the array of langs available - lang = [] - - # loop over each filter - for f in filters: - for filter_key, param in f.items(): - for param_key, param_value in param.items(): - # special cases - if param_key == 'available_locales': # lang is determined by the available locales for the document - if isinstance(param_value, list): - lang = param_value - else: - lang = [param_value] - elif param_key == 'geom': - col = getattr(DocumentGeometry, 'geom') - polygon = Polygon([ - (param_value['left'], param_value['bottom']), - (param_value['right'], param_value['bottom']), - (param_value['right'], param_value['top']), - (param_value['left'], param_value['top']), - (param_value['left'], param_value['bottom']) - ]) - polygon_wkb = from_shape(polygon, srid=4326) - - filter_conditions.append(ST_Intersects( - ST_Transform(col, 4326), polygon_wkb)) - elif param_key in filter_map: # param_key is 'area' - col = getattr(filter_map[param_key], 'document_id') - if isinstance(param_value, list): - filter_conditions.append(col.any(param_value)) - else: - filter_conditions.append(col == param_value) - else: # all filters on Waypoints - # col <=> Waypoint.param_key - col = getattr(Waypoint, param_key) - column = col.property.columns[0] - col_type = column.type - - if filter_key == 'range': - # lte and gte are integers - gte = param_value.get('gte') - lte = param_value.get('lte') - mapper = sortable_search_attr_by_field.get(param_key) - if mapper is not None: - values = [] - if gte is not None and lte is not None: - if gte == lte: - values = [ - val for val in mapper if mapper[val] == gte and mapper[val] == lte] - else: - # find array of possible values (not integers but enum values) between gte and lte - values = [ - val for val in mapper if mapper[val] >= gte and mapper[val] < lte] - - elif gte is not None: - # find array of possible values (not integers but enum values) >= gte - values = [ - val for val in mapper if mapper[val] >= gte] - - elif lte is not None: - # find array of possible values (not integers but enum values) < lte - values = [ - val for val in mapper if mapper[val] < lte] - - # then compare (==) col with each value - # combine multiple checks with | - checks = [(col == val) for val in values] - if len(checks) > 0: - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check - - filter_conditions.append(or_expr) - else: - if gte is not None and lte is not None: - filter_conditions.append(and_(col > gte, col < lte)) - elif gte is not None: - filter_conditions.append(col > gte) - elif lte is not None: - filter_conditions.append(col < lte) - - elif filter_key == 'terms': - values = param_value if isinstance( - param_value, (list, tuple)) else [param_value] - - if isinstance(col_type, ArrayOfEnum): - # combine multiple checks with | - checks = [col.any(v) for v in values] - or_expr = checks[0] - for check in checks[1:]: - or_expr = or_expr | check - filter_conditions.append(or_expr) - else: - filter_conditions.append(col.in_(values)) - - elif filter_key == 'term': - if isinstance(col_type, ArrayOfEnum): - filter_conditions.append(col.any(param_value)) - else: - filter_conditions.append(col == param_value) - - else: - continue - - # combine all conditions with & - if len(filter_conditions) == 0: - filter_conditions = True - elif len(filter_conditions) == 1: - filter_conditions = filter_conditions[0] - else: - final_expr = filter_conditions[0] - for cond in filter_conditions[1:]: - final_expr = final_expr & cond - filter_conditions = final_expr - - # get sort information - sorts = search_dict.get('sort', []) - # compute sort expressions - sort_expressions = [] - for sort in sorts: - if (sort == 'undefined'): - pass - # sort by desc - if (hasattr(sort, 'items')): - for attribute, order in sort.items(): - if (attribute == 'id'): - sort_expressions.append( - nullslast(getattr(Waypoint, 'document_id').desc())) - else: - sort_expressions.append( - nullslast(getattr(Waypoint, attribute).desc())) - else: - # sort by asc - sort_expressions.append(nullslast(getattr(Waypoint, sort).asc())) - # perform query query = DBSession.query(Waypoint, func.jsonb_agg(func.distinct( func.jsonb_build_object( literal_column("'document_id'"), Area.document_id ))).label("areas")). \ select_from(Association). \ - join(Waypoint, or_( - Waypoint.document_id == Association.child_document_id, - Waypoint.document_id == Association.parent_document_id - )). \ - join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ - join(DocumentLocale, Waypoint.document_id == DocumentLocale.document_id). \ - filter(Waypoint.waypoint_type == 'access'). \ + join(Waypoint, + and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' + ) + ). \ join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id )). \ - join(Area, Area.document_id == AreaAssociation.area_id) - - if (len(lang) > 0): - query = query.filter(DocumentLocale.lang.in_(lang)) + join(Area, Area.document_id == AreaAssociation.area_id). \ + join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id) + + if (needs_locale_join): + query = query.join(DocumentLocale, Waypoint.document_id == DocumentLocale.document_id) + + if (len(langs) > 0): + query = query.filter(DocumentLocale.lang.in_(langs)) query = query. \ filter(filter_conditions). \ From 479e003cde226fe31a6640bfec704a6d383cc0ef Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Tue, 2 Dec 2025 12:17:56 +0100 Subject: [PATCH 22/41] [fix] fix issue when filtering with several areas --- c2corg_api/search/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index f42d3b86f..26aab01ec 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -124,7 +124,12 @@ def build_sqlalchemy_filters( elif param_key in filter_map: col = getattr(filter_map[param_key], "document_id") if isinstance(param_value, list): - filter_conditions.append(col.any(param_value)) + checks = [col == v for v in param_value] + if checks: + or_expr = checks[0] + for check in checks[1:]: + or_expr = or_expr | check + filter_conditions.append(or_expr) else: filter_conditions.append(col == param_value) @@ -160,8 +165,6 @@ def build_sqlalchemy_filters( # combine and conditions final_filter = combine_conditions(filter_conditions) - - log.warning(final_filter) # build sort expressions sort_expressions = build_sort_expressions( From 2a1a84739b36926130076078e44d9e1f367bc698 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 3 Dec 2025 10:33:42 +0100 Subject: [PATCH 23/41] [new feature] added route to get coverage for area / area in isochrone --- c2corg_api/views/coverage.py | 52 +++++++++++++++++++++++++++++++----- c2corg_api/views/navitia.py | 32 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/c2corg_api/views/coverage.py b/c2corg_api/views/coverage.py index bc784655a..9a7e32269 100644 --- a/c2corg_api/views/coverage.py +++ b/c2corg_api/views/coverage.py @@ -1,8 +1,12 @@ import functools +import json import logging from operator import and_, or_ from c2corg_api.models import DBSession, coverage +from shapely import transform +from pyproj import Transformer +from c2corg_api.models.area import Area, schema_area from c2corg_api.models.association import Association from c2corg_api.models.common.fields_coverage import fields_coverage from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, schema_coverage, schema_create_coverage, schema_update_coverage @@ -14,7 +18,7 @@ from c2corg_api.views.validation import validate_cook_param, validate_id, validate_lang_param from c2corg_api.views.document import DocumentRest, make_validator_create, make_validator_update from shapely import wkb -from shapely.geometry import Point +from shapely.geometry import Point, shape from cornice.validators import colander_body_validator from cornice.resource import resource, view from sqlalchemy import func @@ -45,7 +49,7 @@ def __init__(self, request, context=None): @view(validators=[validate_pagination, validate_preferred_lang_param]) def collection_get(self): return self._collection_get(COVERAGE_TYPE, coverage_documents_config) - + @view(validators=[validate_id, validate_lang_param, validate_cook_param]) def get(self): return self._get( @@ -77,7 +81,7 @@ class WaypointCoverageRest(DocumentRest): def __init__(self, request, context=None): self.request = request - @view(validators=[validate_pagination, validate_preferred_lang_param]) + @view(validators=[]) def get(self): """Returns the coverage from a longitude and a latitude""" @@ -86,7 +90,23 @@ def get(self): return get_coverage(lon, lat) + +@resource(path='/getpolygoncoverage', cors_policy=cors_policy) +class PolygonCoverage(DocumentRest): + + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def post(self): + """Returns the coverages from a geom_detail type polygon (geom_detail has to be EPSG 4326 since isochrone is 4326)""" + geom_detail = json.loads((json.loads(self.request.body)['geom_detail'])) + polygon = shape(geom_detail) + return get_coverages(polygon) + + def get_coverage(lon, lat): + """get the coverage that contains a point(lon, lat)""" pt = Point(lon, lat) coverageFound = None @@ -95,10 +115,10 @@ def get_coverage(lon, lat): for coverage in coverages: geom = coverage.geometry.geom_detail - + # convert WKB → Shapely polygon poly = wkb_to_shape(geom) - + if poly.contains(pt): coverageFound = coverage break @@ -106,4 +126,24 @@ def get_coverage(lon, lat): if (coverageFound): return coverageFound.coverage_type else: - return None \ No newline at end of file + return None + +def get_coverages(polygon): + """get all the coverages that intersects a polygon""" + coverageFound = [] + + coverages = DBSession.query(Coverage).all() + + for coverage in coverages: + geom = coverage.geometry.geom_detail + + # convert WKB → Shapely polygon + poly = wkb_to_shape(geom) + log.warning(poly) + log.warning(polygon) + + if poly.contains(polygon) or poly.intersects(polygon): + log.warning("coverage found and added") + coverageFound.append(coverage.coverage_type) + + return coverageFound diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index 2fe0a567d..d784194cb 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -5,10 +5,15 @@ from c2corg_api.models import DBSession from c2corg_api.models.area import Area, schema_area from c2corg_api.models.coverage import Coverage +from c2corg_api.models.document import DocumentGeometry from c2corg_api.models.utils import wkb_to_shape from c2corg_api.models.waypoint import Waypoint, schema_waypoint from c2corg_api.views.coverage import get_coverage from c2corg_api.views.document import LIMIT_DEFAULT +from shapely.geometry import Polygon +from geoalchemy2.shape import from_shape +from sqlalchemy import nullslast +from geoalchemy2.functions import ST_Intersects, ST_Transform from c2corg_api.views.waypoint import build_reachable_waypoints_query from c2corg_api.views.route import build_reachable_route_query, build_reachable_route_query_with_waypoints from shapely import wkb @@ -390,6 +395,33 @@ def get(self): return {'documents': waypoints, 'total': len(waypoints), "isochron_geom": geojson} +@resource(path='/navitia/areainisochrone', cors_policy=cors_policy) +class AreaInIsochroneRest: + def __init__(self, request, context=None): + self.request = request + + @view(validators=[]) + def post(self): + """ + returns all areas that are inside or that intersects an isochrone geometry + make sure the geom_detail in body is epsg:3857 + """ + polygon = shape(json.loads(json.loads(self.request.body)['geom_detail'])) + + query = ( + DBSession.query(Area) + ) + + results = query.all() + + areas = [] + + for area in results: + if (polygon.intersects(wkb_to_shape(area.geometry.geom_detail))): + areas.append(area.document_id) + + return areas + def is_wp_journey_reachable(waypoint, journey_params): """ From 2bd8db5f70ab7158c337ed5a76aabc874161aa98 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 3 Dec 2025 10:34:10 +0100 Subject: [PATCH 24/41] [fix] fixed a bug where bbox applied to routes was not done on the access waypoint of the route --- c2corg_api/views/route.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 21c22cb9b..b4965ef61 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -643,7 +643,7 @@ def build_reachable_route_query(params, meta_params): Waypoint.waypoint_type == 'access' )). \ join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ - join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ + join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id @@ -709,7 +709,7 @@ def build_reachable_route_query_with_waypoints(params, meta_params): Waypoint.waypoint_type == 'access' )). \ join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ - join(DocumentGeometry, Route.document_id == DocumentGeometry.document_id). \ + join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id From 4950c16b063b01dad654fe4efc10bbc17743500b Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 3 Dec 2025 11:26:49 +0100 Subject: [PATCH 25/41] [lint] flake8 linting --- c2corg_api/__init__.py | 4 +- c2corg_api/models/coverage.py | 8 +- .../scripts/migration/documents/coverage.py | 8 +- c2corg_api/search/utils.py | 51 +++-- c2corg_api/views/coverage.py | 60 +++--- c2corg_api/views/document_revert.py | 1 - c2corg_api/views/document_schemas.py | 3 +- c2corg_api/views/navitia.py | 198 +++++++++++------- c2corg_api/views/route.py | 110 +++++----- c2corg_api/views/waypoint.py | 78 ++++--- 10 files changed, 301 insertions(+), 220 deletions(-) diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index b08c20bf5..67105d915 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -125,7 +125,7 @@ def process_new_waypoint(mapper, connection, geometry): inserting it into documents_geometries.""" # Check if document is a waypoint waypoint_id = geometry.document_id - + document_type = connection.execute( text( """ @@ -138,7 +138,7 @@ def process_new_waypoint(mapper, connection, geometry): if document_type != "w": return - + log.debug("Entering process_new_waypoint callback") max_distance_waypoint_to_stoparea = int( os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") diff --git a/c2corg_api/models/coverage.py b/c2corg_api/models/coverage.py index 6fe17d861..17004c7e3 100644 --- a/c2corg_api/models/coverage.py +++ b/c2corg_api/models/coverage.py @@ -1,7 +1,11 @@ from c2corg_api.models import Base, schema from c2corg_api.models.enums import coverage_types from c2corg_api.models.document import ( - ArchiveDocument, Document, get_geometry_schema_overrides, schema_document_locale, schema_attributes) + schema_document_locale, + ArchiveDocument, + Document, + get_geometry_schema_overrides, + schema_attributes) from c2corg_api.models.schema_utils import get_update_schema, \ get_create_schema, restrict_schema from c2corg_api.models.utils import copy_attributes @@ -51,6 +55,7 @@ def update(self, other): schema_coverage_locale = schema_document_locale schema_coverage_attributes = list(schema_attributes) + class ArchiveCoverage(_CoverageMixin, ArchiveDocument): """ """ @@ -67,6 +72,7 @@ class ArchiveCoverage(_CoverageMixin, ArchiveDocument): __table_args__ = Base.__table_args__ + schema_coverage = SQLAlchemySchemaNode( Coverage, # whitelisted attributes diff --git a/c2corg_api/scripts/migration/documents/coverage.py b/c2corg_api/scripts/migration/documents/coverage.py index 94b403917..aa53aa0ed 100644 --- a/c2corg_api/scripts/migration/documents/coverage.py +++ b/c2corg_api/scripts/migration/documents/coverage.py @@ -47,7 +47,8 @@ def get_query(self): def get_count_query_locales(self): return ( 'select count(*) ' - 'from app_coverages_i18n_archives aa join coverages a on aa.id = a.id ' + 'from app_coverages_i18n_archives aa ' + 'join coverages a on aa.id = a.id ' 'where a.redirects_to is null;' ) @@ -56,7 +57,8 @@ def get_query_locales(self): 'select ' ' aa.id, aa.document_i18n_archive_id, aa.is_latest_version, ' ' aa.culture, aa.name, aa.description ' - 'from app_coverages_i18n_archives aa join coverages a on aa.id = a.id ' + 'from app_coverages_i18n_archives aa ' + 'join coverages a on aa.id = a.id ' 'where a.redirects_to is null ' 'order by aa.id, aa.culture, aa.document_i18n_archive_id;' ) @@ -85,4 +87,4 @@ def get_document_locale(self, document_in, version): title=document_in.name, description=description, summary=summary - ) \ No newline at end of file + ) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index 26aab01ec..540f4108f 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -43,25 +43,32 @@ def get_title(title, title_prefix): def build_sqlalchemy_filters( - search_dict, # elastic search dict - document_model, # the model (waypoint, routes, etc...) - filter_map, # for multicriteria search (ex : searching a waypoint by area id) - geometry_model, # the Geometry model (where ce access to geometry), most likely always DocumentGeometry - range_enum_map, # the mapper for range enum, most likely always sortable_search_attr_by_field - title_columns=None # the column for the title (ex: Waypoint -> title, Route -> title and title_prefix) + search_dict, # elastic search dict + document_model, # the model (waypoint, routes, etc...) + # for multicriteria search (ex : searching a waypoint by area id) + filter_map, + # the Geometry model (where ce access to geometry) + geometry_model, # most likely always DocumentGeometry + # the mapper for range enum + range_enum_map, # most likely always sortable_search_attr_by_field + # the column for the title + # (ex: Waypoint -> title, Route -> title and title_prefix) + title_columns=None ): """ - Build SQLAlchemy filter for documents (Waypoint, Route, etc.) based on filters that would normally be used by ElasticSearch - + Build SQLAlchemy filter for documents (Waypoint, Route, etc.) + based on filters that would normally be used by ElasticSearch + this can then be used to filter directly in a DB query - - Usage Example : - + + Usage Example : + search = build_query(params, meta_params, WAYPOINT_TYPE) search_dict = search.to_dict() - - filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( + + filter_conditions, sort_expressions, needs_locale_join, langs = + build_sqlalchemy_filters( search_dict=search_dict, document_model=Waypoint, filter_map={"areas": Area,}, @@ -69,9 +76,11 @@ def build_sqlalchemy_filters( range_enum_map=sortable_search_attr_by_field, title_columns=[DocumentLocale.title] ) - - query = DBSession.query(Waypoint).filter(filter_conditions).order_by(*sort_expressions) - + + query = DBSession.query(Waypoint) + .filter(filter_conditions) + .order_by(*sort_expressions) + """ filters = search_dict.get("query", {}).get("bool", {}).get("filter", []) @@ -143,7 +152,8 @@ def build_sqlalchemy_filters( if filter_key == "range": filter_conditions.append( build_range_expression( - col, param_value, range_enum_map.get(param_key)) + col, param_value, range_enum_map.get(param_key) + ) ) # for terms @@ -171,7 +181,7 @@ def build_sqlalchemy_filters( search_dict.get("sort", []), document_model ) - # return each valuable variable to be used later in a sql alchemy DBSession.query + # return each valuable variable to be used later in a sql alchemy query return final_filter, sort_expressions, needs_locale_join, langs @@ -189,7 +199,8 @@ def build_range_expression(col, param_value, enum_map): if gte == lte: values = [val for val, num in enum_map.items() if num == gte] else: - values = [val for val, num in enum_map.items() if num >= gte and num < lte] + values = [val for val, num in enum_map.items() if num >= + gte and num < lte] elif gte is not None: values = [val for val, num in enum_map.items() if num >= gte] elif lte is not None: @@ -213,6 +224,7 @@ def build_range_expression(col, param_value, enum_map): return False return and_(*clauses) + def build_terms_expression(col, values, col_type): """ build sql alchemy filter for terms expressions @@ -237,6 +249,7 @@ def build_terms_expression(col, values, col_type): return col == values[0] return col.in_(values) + def build_term_expression(col, value, col_type): """ build sql alchemy filter for term expressions diff --git a/c2corg_api/views/coverage.py b/c2corg_api/views/coverage.py index 9a7e32269..9b44943d2 100644 --- a/c2corg_api/views/coverage.py +++ b/c2corg_api/views/coverage.py @@ -1,31 +1,24 @@ +from c2corg_api.views.validation import validate_associations, \ + validate_pagination, validate_preferred_lang_param +from c2corg_api.views import cors_policy, restricted_json_view +from cornice.resource import resource, view +from cornice.validators import colander_body_validator +from shapely.geometry import Point, shape +from c2corg_api.views.document import DocumentRest, make_validator_create, \ + make_validator_update +from c2corg_api.views.validation import validate_cook_param, validate_id, \ + validate_lang_param +from c2corg_api.views.document_schemas import coverage_documents_config +from c2corg_api.models.utils import wkb_to_shape import functools import json import logging -from operator import and_, or_ -from c2corg_api.models import DBSession, coverage -from shapely import transform -from pyproj import Transformer -from c2corg_api.models.area import Area, schema_area -from c2corg_api.models.association import Association +from c2corg_api.models import DBSession from c2corg_api.models.common.fields_coverage import fields_coverage -from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, schema_coverage, schema_create_coverage, schema_update_coverage -from c2corg_api.models.document import DocumentGeometry -from c2corg_api.models.utils import wkb_to_shape -from c2corg_api.models.waypoint import Waypoint -from c2corg_api.views.document_schemas import coverage_documents_config -from c2corg_api.views.area import update_associations -from c2corg_api.views.validation import validate_cook_param, validate_id, validate_lang_param -from c2corg_api.views.document import DocumentRest, make_validator_create, make_validator_update -from shapely import wkb -from shapely.geometry import Point, shape -from cornice.validators import colander_body_validator -from cornice.resource import resource, view -from sqlalchemy import func +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, \ + schema_coverage, schema_create_coverage, schema_update_coverage -from c2corg_api.views import cors_policy, restricted_json_view, to_json_dict -from c2corg_api.views.validation import validate_associations, validate_pagination, \ - validate_preferred_lang_param log = logging.getLogger(__name__) @@ -99,8 +92,10 @@ def __init__(self, request, context=None): @view(validators=[]) def post(self): - """Returns the coverages from a geom_detail type polygon (geom_detail has to be EPSG 4326 since isochrone is 4326)""" - geom_detail = json.loads((json.loads(self.request.body)['geom_detail'])) + """Returns the coverages from a geom_detail type polygon + (geom_detail has to be EPSG 4326 since isochrone is 4326)""" + geom_detail = json.loads( + (json.loads(self.request.body)['geom_detail'])) polygon = shape(geom_detail) return get_coverages(polygon) @@ -109,7 +104,7 @@ def get_coverage(lon, lat): """get the coverage that contains a point(lon, lat)""" pt = Point(lon, lat) - coverageFound = None + coverage_found = None coverages = DBSession.query(Coverage).all() @@ -120,17 +115,18 @@ def get_coverage(lon, lat): poly = wkb_to_shape(geom) if poly.contains(pt): - coverageFound = coverage + coverage_found = coverage break - if (coverageFound): - return coverageFound.coverage_type + if (coverage_found): + return coverage_found.coverage_type else: return None - + + def get_coverages(polygon): """get all the coverages that intersects a polygon""" - coverageFound = [] + coverage_found = [] coverages = DBSession.query(Coverage).all() @@ -144,6 +140,6 @@ def get_coverages(polygon): if poly.contains(polygon) or poly.intersects(polygon): log.warning("coverage found and added") - coverageFound.append(coverage.coverage_type) + coverage_found.append(coverage.coverage_type) - return coverageFound + return coverage_found diff --git a/c2corg_api/views/document_revert.py b/c2corg_api/views/document_revert.py index 5f8243060..7eb0be4ab 100644 --- a/c2corg_api/views/document_revert.py +++ b/c2corg_api/views/document_revert.py @@ -1,6 +1,5 @@ import logging -from c2corg_api.models.coverage import COVERAGE_TYPE, ArchiveCoverage, Coverage from c2corg_api.security.acl import ACLDefault from c2corg_api import DBSession from c2corg_api.models.area import AREA_TYPE, Area, ArchiveArea diff --git a/c2corg_api/views/document_schemas.py b/c2corg_api/views/document_schemas.py index 4f1865786..90a842736 100644 --- a/c2corg_api/views/document_schemas.py +++ b/c2corg_api/views/document_schemas.py @@ -2,7 +2,8 @@ from c2corg_api.models.article import ARTICLE_TYPE, Article, \ schema_listing_article from c2corg_api.models.book import BOOK_TYPE, Book, schema_listing_book -from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, schema_listing_coverage +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage, \ + schema_listing_coverage from c2corg_api.models.image import IMAGE_TYPE, Image, schema_listing_image from c2corg_api.models.outing import OUTING_TYPE, Outing, schema_outing from c2corg_api.models.xreport import XREPORT_TYPE, Xreport, \ diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index d784194cb..0860e24ec 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -3,36 +3,31 @@ import os import requests from c2corg_api.models import DBSession -from c2corg_api.models.area import Area, schema_area -from c2corg_api.models.coverage import Coverage -from c2corg_api.models.document import DocumentGeometry +from c2corg_api.models.area import Area from c2corg_api.models.utils import wkb_to_shape from c2corg_api.models.waypoint import Waypoint, schema_waypoint from c2corg_api.views.coverage import get_coverage from c2corg_api.views.document import LIMIT_DEFAULT -from shapely.geometry import Polygon -from geoalchemy2.shape import from_shape -from sqlalchemy import nullslast -from geoalchemy2.functions import ST_Intersects, ST_Transform from c2corg_api.views.waypoint import build_reachable_waypoints_query -from c2corg_api.views.route import build_reachable_route_query, build_reachable_route_query_with_waypoints -from shapely import wkb +from c2corg_api.views.route import build_reachable_route_query_with_waypoints from shapely.geometry import Point from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError # noqa: E501 from cornice.resource import resource, view from c2corg_api.views import cors_policy, to_json_dict -from c2corg_api.models.route import Route, schema_route +from c2corg_api.models.route import schema_route from c2corg_api.models.area import schema_listing_area from shapely.geometry import shape from pyproj import Transformer log = logging.getLogger(__name__) -# When editing these constants, make sure to edit them in the front too (itinevert-service) +# When editing these constants, make sure to edit them in the front too +# (itinevert-service) MAX_ROUTE_THRESHOLD = 50 MAX_TRIP_DURATION = 240 MIN_TRIP_DURATION = 20 + def validate_navitia_params(request, **kwargs): """Validates the required parameters for the Navitia API""" required_params = ['from', 'to', 'datetime', 'datetime_represents'] @@ -137,7 +132,8 @@ def get(self): def validate_journey_reachable_params(request, **kwargs): """Validates the required parameters for the journey reachable doc route""" - required_params = ['from', 'datetime', 'datetime_represents', 'walking_speed', 'max_walking_duration_to_pt'] + required_params = ['from', 'datetime', 'datetime_represents', + 'walking_speed', 'max_walking_duration_to_pt'] for param in required_params: if param not in request.params: @@ -146,6 +142,7 @@ def validate_journey_reachable_params(request, **kwargs): param, f'Paramètre {param} requis') + @resource(path='/navitia/journeyreachableroutes', cors_policy=cors_policy) class NavitiaJourneyReachableRoutesRest: def __init__(self, request, context=None): @@ -154,8 +151,13 @@ def __init__(self, request, context=None): @view(validators=[validate_journey_reachable_params]) def get(self): """ - Get all routes matching filters in params, that are reachable (means there exists a Navitia journey for at least one of their waypoints of type access). - NOTE : the number of routes after applying filters, has to be < MAX_ROUTE_THRESHOLD, to reduce number of queries towards Navitia journey API + Get all routes matching filters in params, that are reachable + (means there exists a Navitia journey for at least one of + their waypoints of type access). + + NOTE : the number of routes after applying filters, + has to be < MAX_ROUTE_THRESHOLD, + to reduce number of queries towards Navitia journey API """ meta_params = extract_meta_params(self.request) @@ -167,12 +169,16 @@ def get(self): results = ( query.all() ) - + # /!\ IMPORTANT - # Before doing any further computations, make sure the number of routes don't go above threshold. - # The Itinevert UI doesn't allow that, but a user could query the api with wrong parameters... + # Before doing any further computations, + # make sure the number of routes don't go above threshold. + # The Itinevert UI doesn't allow that, + # but a user could query the api with wrong parameters... if len(results) > MAX_ROUTE_THRESHOLD: - raise HTTPBadRequest("Couldn't proceed with the computation : Too much routes found.") + raise HTTPBadRequest( + "Couldn't proceed with computation : Too much routes found." + ) areas_map = collect_areas_from_results(results, 1) @@ -180,12 +186,15 @@ def get(self): log.info("Number of NAVITIA journey queries : %d", len(wp_objects)) - navitia_wp_map = {wp.document_id: is_wp_journey_reachable( - to_json_dict(wp, schema_waypoint), journey_params) for wp in wp_objects} + navitia_wp_map = { + wp.document_id: is_wp_journey_reachable( + to_json_dict(wp, schema_waypoint), journey_params + ) for wp in wp_objects} routes = [] for route, areas, waypoints in results: - # check if a journey exists for route (at least one wp has a journey associated) + # check if a journey exists for route + # (at least one wp has a journey associated) journey_exists = False for wp in waypoints: wp_id = wp.get("document_id") @@ -209,6 +218,7 @@ def get(self): return {'documents': routes, 'total': len(routes)} + @resource(path='/navitia/journeyreachablewaypoints', cors_policy=cors_policy) class NavitiaJourneyReachableWaypointsRest: def __init__(self, request, context=None): @@ -217,17 +227,19 @@ def __init__(self, request, context=None): @view(validators=[validate_journey_reachable_params]) def get(self): """ - Get all waypoints matching filters in params, that are reachable (means there exists a Navitia journey). - NOTE : waypoints should be filtered with one area, to reduce the number of queries towards Navitia journey API. + Get all waypoints matching filters in params, that are reachable + (means there exists a Navitia journey). + NOTE : waypoints should be filtered with one area, + to reduce the number of queries towards Navitia journey API. """ meta_params = extract_meta_params(self.request) journey_params = extract_journey_params(self.request) - + areas = None try: areas = self.request.GET['a'].split(",") - except Exception as e: + except Exception: areas = None # Normalize: allow single value or list @@ -251,7 +263,9 @@ def get(self): waypoints = [] for waypoint, areas in results: # check if a journey exists for waypoint - if is_wp_journey_reachable(to_json_dict(waypoint, schema_waypoint), journey_params): + if is_wp_journey_reachable( + to_json_dict(waypoint, schema_waypoint), journey_params + ): json_areas = [] if areas is None: areas = [] @@ -269,9 +283,12 @@ def get(self): return {'documents': waypoints, 'total': len(waypoints)} + def validate_isochrone_reachable_params(request, **kwargs): - """Validates the required parameters for the isochrone reachable doc route""" - required_params = ['from', 'datetime', 'datetime_represents', 'boundary_duration'] + """Validates the required parameters + for the isochrone reachable doc route""" + required_params = ['from', 'datetime', + 'datetime_represents', 'boundary_duration'] for param in required_params: if param not in request.params: @@ -280,6 +297,7 @@ def validate_isochrone_reachable_params(request, **kwargs): param, f'Paramètre {param} requis') + @resource(path='/navitia/isochronesreachableroutes', cors_policy=cors_policy) class NavitiaIsochronesReachableRoutesRest: def __init__(self, request, context=None): @@ -288,8 +306,10 @@ def __init__(self, request, context=None): @view(validators=[validate_isochrone_reachable_params]) def get(self): """ - Get all routes matching filters in params, that have at least one waypoint of type access that is inside the isochron. - The isochron is created by querying navitia api with specific parameters, see validate_isochrone_reachable_params func + Get all routes matching filters in params, that have at least + one waypoint of type access that is inside the isochron. + The isochron is created by querying navitia api + with specific parameters, see validate_isochrone_reachable_params func """ meta_params = extract_meta_params(self.request) @@ -303,27 +323,29 @@ def get(self): areas_map = collect_areas_from_results(results, 1) response = get_navitia_isochrone(isochrone_params) - + routes = [] geojson = "" # if isochrone found if (len(response["isochrones"]) > 0): geojson = response["isochrones"][0]["geojson"] isochrone_geom = shape(geojson) - + wp_objects = collect_waypoints_from_results(results) navitia_wp_map = {wp.document_id: is_wp_in_isochrone( - to_json_dict(wp, schema_waypoint), isochrone_geom) for wp in wp_objects} + to_json_dict(wp, schema_waypoint), isochrone_geom + ) for wp in wp_objects} for route, areas, waypoints in results: - # check if a journey exists for route (at least one wp has a journey associated) + # check if a journey exists for route + # (at least one wp has a journey associated) one_wp_in_isochrone = False for wp in waypoints: wp_id = wp.get("document_id") one_wp_in_isochrone |= navitia_wp_map.get(wp_id) - if one_wp_in_isochrone: + if one_wp_in_isochrone: json_areas = [] if areas is None: @@ -340,10 +362,17 @@ def get(self): route_dict = to_json_dict(route, schema_route, True) routes.append(route_dict) - return {'documents': routes, 'total': len(routes), 'isochron_geom': geojson} + return { + 'documents': routes, + 'total': len(routes), + 'isochron_geom': geojson + } -@resource(path='/navitia/isochronesreachablewaypoints', cors_policy=cors_policy) +@resource( + path='/navitia/isochronesreachablewaypoints', + cors_policy=cors_policy +) class NavitiaIsochronesReachableWaypointsRest: def __init__(self, request, context=None): self.request = request @@ -351,8 +380,10 @@ def __init__(self, request, context=None): @view(validators=[validate_isochrone_reachable_params]) def get(self): """ - Get all waypoints matching filters in params, that are inside the isochron. - The isochron is created by querying navitia api with specific parameters, see validate_isochrone_reachable_params func + Get all waypoints matching filters in params, + that are inside the isochron. + The isochron is created by querying navitia api + with specific parameters, see validate_isochrone_reachable_params func """ meta_params = extract_meta_params(self.request) @@ -367,7 +398,7 @@ def get(self): areas_map = collect_areas_from_results(results, 1) response = get_navitia_isochrone(isochrone_params) - + waypoints = [] geojson = "" # if isochrone found @@ -377,7 +408,9 @@ def get(self): for waypoint, areas in results: # check if wp is in isochrone - if is_wp_in_isochrone(to_json_dict(waypoint, schema_waypoint), isochrone_geom): + if is_wp_in_isochrone( + to_json_dict(waypoint, schema_waypoint), isochrone_geom + ): json_areas = [] if areas is None: areas = [] @@ -393,7 +426,12 @@ def get(self): wp_dict = to_json_dict(waypoint, schema_waypoint, True) waypoints.append(wp_dict) - return {'documents': waypoints, 'total': len(waypoints), "isochron_geom": geojson} + return { + 'documents': waypoints, + 'total': len(waypoints), + "isochron_geom": geojson + } + @resource(path='/navitia/areainisochrone', cors_policy=cors_policy) class AreaInIsochroneRest: @@ -403,32 +441,38 @@ def __init__(self, request, context=None): @view(validators=[]) def post(self): """ - returns all areas that are inside or that intersects an isochrone geometry + returns all areas that are inside + or that intersects an isochrone geometry + make sure the geom_detail in body is epsg:3857 """ - polygon = shape(json.loads(json.loads(self.request.body)['geom_detail'])) - + polygon = shape(json.loads(json.loads( + self.request.body)['geom_detail'])) + query = ( DBSession.query(Area) ) - + results = query.all() - + areas = [] - + for area in results: if (polygon.intersects(wkb_to_shape(area.geometry.geom_detail))): areas.append(area.document_id) - + return areas - + def is_wp_journey_reachable(waypoint, journey_params): """ - Query the navitia Journey api and returns true if the waypoint is reachable (at least one journey has been found) - NOTE : the journey's departure time has to be the same day as the datetime's day in journey_params + Query the navitia Journey api and returns true + if the waypoint is reachable (at least one journey has been found) + NOTE : the journey's departure time has to be + the same day as the datetime's day in journey_params """ - # enhance journey params with the 'to' parameter, from the waypoint geometry. + # enhance journey params with the 'to' parameter, + # from the waypoint geometry. geom = shape(json.loads(waypoint.get("geometry").get("geom"))) src_epsg = 3857 @@ -451,7 +495,7 @@ def is_wp_journey_reachable(waypoint, journey_params): if (destination_coverage): # Appel à l'API Navitia Journey with coverage response = requests.get( - f'https://api.navitia.io/v1/coverage/{destination_coverage}/journeys', + f'https://api.navitia.io/v1/coverage/{destination_coverage}/journeys', # noqa: E501 params=journey_params, headers={'Authorization': api_key}, timeout=30 @@ -475,20 +519,21 @@ def is_wp_journey_reachable(waypoint, journey_params): elif not response.ok: return False - # make sure the waypoint is reachable if at least one journey's departure date time is the same day as the day in journey_params + # make sure the waypoint is reachable if at least one journey's + # departure date time is the same day as the day in journey_params for journey in response.json()['journeys']: journey_day = int(journey['departure_date_time'][6:8]) param_day = int(journey_params['datetime'][6:8]) if journey_day == param_day: return True - + return False except requests.exceptions.Timeout: return False - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: return False - except Exception as e: + except Exception: return False @@ -512,7 +557,7 @@ def get_navitia_isochrone(isochrone_params): if (source_coverage): # Appel à l'API Navitia Journey with coverage response = requests.get( - f'https://api.navitia.io/v1/coverage/{source_coverage}/isochrones', + f'https://api.navitia.io/v1/coverage/{source_coverage}/isochrones', # noqa: E501 params=isochrone_params, headers={'Authorization': api_key}, timeout=30 @@ -557,10 +602,10 @@ def is_wp_in_isochrone(waypoint, isochrone_geom): f"EPSG:{src_epsg}", "EPSG:4326", always_xy=True) lon, lat = transformer.transform(geom.x, geom.y) pt = Point(lon, lat) - + return isochrone_geom.contains(pt) - - + + def extract_meta_params(request): """ Extract meta parameters such as offset, limit and lang @@ -582,7 +627,7 @@ def extract_journey_params(request): 'datetime': request.params.get('datetime'), 'datetime_represents': request.params.get('datetime_represents'), 'walking_speed': request.params.get('walking_speed'), - 'max_walking_duration_to_pt': request.params.get('max_walking_duration_to_pt'), + 'max_walking_duration_to_pt': request.params.get('max_walking_duration_to_pt'), # noqa: E501 'to': '' } @@ -590,8 +635,11 @@ def extract_journey_params(request): def extract_isochrone_params(request): """ Extract parameters for isochrone query - NOTE : the boundary duration is bounded by constants MAX_TRIP_DURATION and MIN_TRIP_DURATION - if the boundary duration goes beyond limits, it is set to the limit it goes past. + + NOTE : the boundary duration is bounded by constants + MAX_TRIP_DURATION and MIN_TRIP_DURATION + if the boundary duration goes beyond limits, + it is set to the limit it goes past. """ params = { 'from': request.params.get('from'), @@ -603,11 +651,17 @@ def extract_isochrone_params(request): bd = params['boundary_duration[]'] if len(bd.split(",")) == 1: duration = int(bd) - params['boundary_duration[]'] = max(min(duration, MAX_TRIP_DURATION * 60), - MIN_TRIP_DURATION * 60) + params['boundary_duration[]'] = max( + min( + duration, + MAX_TRIP_DURATION * 60 + ), + MIN_TRIP_DURATION * 60 + ) return params -def collect_areas_from_results(results, areaIndex): + +def collect_areas_from_results(results, area_index): """ Extract all area document_ids from results, load Area objects from DB, and return {document_id: Area}. @@ -615,7 +669,7 @@ def collect_areas_from_results(results, areaIndex): area_ids = set() for row in results: - areas = row[areaIndex] + areas = row[area_index] if not areas: continue @@ -631,9 +685,11 @@ def collect_areas_from_results(results, areaIndex): return {a.document_id: a for a in area_objects} + def collect_waypoints_from_results(results): """ - Extract all waypoint document_ids from results, load Waypoint objects from DB, + Extract all waypoint document_ids from results, + load Waypoint objects from DB, and return {document_id: Waypoint}. """ wp_ids = set() @@ -651,4 +707,4 @@ def collect_waypoints_from_results(results): Waypoint.document_id.in_(wp_ids) ).all() - return {wp for wp in wp_objects} \ No newline at end of file + return {wp for wp in wp_objects} diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index b4965ef61..7e5706fa7 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -35,31 +35,17 @@ from sqlalchemy.orm import load_only from sqlalchemy.sql.expression import text, or_, column, union -from operator import and_, or_ +from operator import and_ from c2corg_api.models.area import Area from c2corg_api.models.area_association import AreaAssociation -from c2corg_api.models.association import Association -from c2corg_api.models.document import DocumentGeometry, DocumentLocale -from c2corg_api.models.utils import ArrayOfEnum from c2corg_api.search.search_filters import build_query -from c2corg_api.views.validation import validate_pagination, validate_preferred_lang_param -from c2corg_api.views import cors_policy, to_json_dict -from c2corg_api.views.document import ( - LIMIT_DEFAULT, DocumentRest) -from c2corg_api.models.waypoint_stoparea import ( - WaypointStoparea) -from c2corg_api.models import DBSession -from c2corg_api.models.route import ROUTE_TYPE, Route -from c2corg_api.models.waypoint import Waypoint +from c2corg_api.views import to_json_dict +from c2corg_api.views.document import (LIMIT_DEFAULT) +from c2corg_api.models.waypoint_stoparea import (WaypointStoparea) from c2corg_api.models.area import schema_listing_area -from c2corg_api.models.route import schema_route -from shapely.geometry import Polygon -from geoalchemy2.shape import from_shape from sqlalchemy import func, literal_column -from geoalchemy2.functions import ST_Intersects, ST_Transform -from cornice.resource import resource, view -from c2corg_api.models.common.sortable_search_attributes import sortable_search_attr_by_field -from sqlalchemy import nullslast +from c2corg_api.models.common.sortable_search_attributes import \ + sortable_search_attr_by_field validate_route_create = make_validator_create( @@ -240,7 +226,7 @@ def __init__(self, request, context=None): def get(self): """Returns a list of object {documents: Route[], total: Integer} -> documents: routes reachable within offset and limit - total: number of documents returned by query without offset and limit""" + total: number of documents returned by query without offset and limit""" # noqa: E501 validated = self.request.validated @@ -610,20 +596,23 @@ def worst_rating(rating1, rating2): def build_reachable_route_query(params, meta_params): """build the query based on params and meta params. - this includes every filters on route, as well as offset + limit, sort, bbox... - returns a list of routes reachable (can be accessible by public transports), filtered with params + this includes every filters on route, + as well as offset + limit, sort, bbox... + returns a list of routes reachable + (can be accessible by public transports), filtered with params """ search = build_query(params, meta_params, ROUTE_TYPE) search_dict = search.to_dict() - - filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( - search_dict, - document_model=Route, - filter_map={'areas': Area, 'waypoints': Waypoint}, - geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, - title_columns=[DocumentLocale.title, RouteLocale.title_prefix] - ) + + filter_conditions, sort_expressions, needs_locale_join, langs = \ + build_sqlalchemy_filters( + search_dict, + document_model=Route, + filter_map={'areas': Area, 'waypoints': Waypoint}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title, RouteLocale.title_prefix] + ) # perform query query = DBSession.query(Route, func.jsonb_agg(func.distinct( @@ -642,8 +631,14 @@ def build_reachable_route_query(params, meta_params): ), Waypoint.waypoint_type == 'access' )). \ - join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ - join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id + ). \ + join( + DocumentGeometry, + Waypoint.document_id == DocumentGeometry.document_id + ). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id @@ -652,7 +647,10 @@ def build_reachable_route_query(params, meta_params): if (needs_locale_join): query = query. \ - join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ + join( + DocumentLocale, + Route.document_id == DocumentLocale.document_id + ). \ join(RouteLocale, RouteLocale.id == DocumentLocale.id) if (len(langs) > 0): @@ -669,21 +667,24 @@ def build_reachable_route_query(params, meta_params): def build_reachable_route_query_with_waypoints(params, meta_params): """build the query based on params and meta params. - this includes every filters on route, as well as offset + limit, sort, bbox... - returns a list of routes reachable (accessible by common transports), filtered with params + this includes every filters on route, + as well as offset + limit, sort, bbox... + returns a list of routes reachable + (accessible by common transports), filtered with params """ search = build_query(params, meta_params, ROUTE_TYPE) search_dict = search.to_dict() - - filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( - search_dict, - document_model=Route, - filter_map={'areas': Area, 'waypoints': Waypoint}, - geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, - title_columns=[DocumentLocale.title, RouteLocale.title_prefix] - ) - + + filter_conditions, sort_expressions, needs_locale_join, langs = \ + build_sqlalchemy_filters( + search_dict, + document_model=Route, + filter_map={'areas': Area, 'waypoints': Waypoint}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title, RouteLocale.title_prefix] + ) + # perform query query = DBSession.query(Route, func.jsonb_agg(func.distinct( @@ -708,8 +709,14 @@ def build_reachable_route_query_with_waypoints(params, meta_params): ), Waypoint.waypoint_type == 'access' )). \ - join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ - join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id). \ + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id + ). \ + join( + DocumentGeometry, + Waypoint.document_id == DocumentGeometry.document_id + ). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id @@ -718,7 +725,10 @@ def build_reachable_route_query_with_waypoints(params, meta_params): if (needs_locale_join): query = query. \ - join(DocumentLocale, Route.document_id == DocumentLocale.document_id). \ + join( + DocumentLocale, + Route.document_id == DocumentLocale.document_id + ). \ join(RouteLocale, RouteLocale.id == DocumentLocale.id) if (len(langs) > 0): diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index 232ccf175..0ddb83a9c 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -36,30 +36,20 @@ from sqlalchemy.orm.util import aliased from sqlalchemy.sql.elements import literal_column from sqlalchemy.sql.expression import and_, text, union, column -from operator import and_, or_ +from operator import or_ from c2corg_api.models.area import Area from c2corg_api.models.area_association import AreaAssociation -from c2corg_api.models.association import Association from c2corg_api.models.document import DocumentGeometry, DocumentLocale -from c2corg_api.models.utils import ArrayOfEnum from c2corg_api.search.search_filters import build_query -from c2corg_api.views.validation import validate_pagination, validate_preferred_lang_param -from c2corg_api.views import cors_policy, to_json_dict +from c2corg_api.views import to_json_dict from c2corg_api.views.document import ( - LIMIT_DEFAULT, DocumentRest) + LIMIT_DEFAULT) from c2corg_api.models.waypoint_stoparea import ( WaypointStoparea) -from c2corg_api.models import DBSession -from c2corg_api.models.route import ROUTE_TYPE, Route -from c2corg_api.models.waypoint import Waypoint from c2corg_api.models.area import schema_listing_area -from shapely.geometry import Polygon -from geoalchemy2.shape import from_shape -from sqlalchemy import func, literal_column -from geoalchemy2.functions import ST_Intersects, ST_Transform -from cornice.resource import resource, view -from c2corg_api.models.common.sortable_search_attributes import sortable_search_attr_by_field -from sqlalchemy import nullslast +from sqlalchemy import func +from c2corg_api.models.common.sortable_search_attributes import \ + sortable_search_attr_by_field log = logging.getLogger(__name__) @@ -431,7 +421,7 @@ def __init__(self, request, context=None): def get(self): """Returns a list of object {documents: Waypoint[], total: Integer} -> documents: waypoints reachable within offset and limit - total: number of documents returned by query without offset and limit""" + total: number of documents returned by query without offset and limit""" # noqa: E501 validated = self.request.validated meta_params = { @@ -555,47 +545,55 @@ def update_linked_routes_public_transportation_rating(waypoint, update_types): def build_reachable_waypoints_query(params, meta_params): """build the query based on params and meta params. - this includes every filters on waypoints, as well as offset + limit, sort, bbox... - returns a list of waypoints reachable (can be accessible by public transports), filtered with params + this includes every filters on waypoints, + as well as offset + limit, sort, bbox... + returns a list of waypoints reachable + (can be accessible by public transports), filtered with params """ search = build_query(params, meta_params, WAYPOINT_TYPE) search_dict = search.to_dict() - filter_conditions, sort_expressions, needs_locale_join, langs = build_sqlalchemy_filters( - search_dict=search_dict, - document_model=Waypoint, - filter_map={"areas": Area,}, - geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, - title_columns=[DocumentLocale.title] - ) - + filter_conditions, sort_expressions, needs_locale_join, langs = \ + build_sqlalchemy_filters( + search_dict=search_dict, + document_model=Waypoint, + filter_map={"areas": Area}, + geometry_model=DocumentGeometry, + range_enum_map=sortable_search_attr_by_field, + title_columns=[DocumentLocale.title] + ) + # perform query query = DBSession.query(Waypoint, func.jsonb_agg(func.distinct( func.jsonb_build_object( literal_column("'document_id'"), Area.document_id ))).label("areas")). \ select_from(Association). \ - join(Waypoint, - and_( - or_( - Waypoint.document_id == Association.child_document_id, - Waypoint.document_id == Association.parent_document_id - ), - Waypoint.waypoint_type == 'access' - ) + join(Waypoint, + and_( + or_( + Waypoint.document_id == Association.child_document_id, + Waypoint.document_id == Association.parent_document_id + ), + Waypoint.waypoint_type == 'access' + ) + ). \ + join( + WaypointStoparea, + WaypointStoparea.waypoint_id == Waypoint.document_id ). \ - join(WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id). \ join(AreaAssociation, or_( AreaAssociation.document_id == Association.child_document_id, AreaAssociation.document_id == Association.parent_document_id )). \ join(Area, Area.document_id == AreaAssociation.area_id). \ - join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id) - + join(DocumentGeometry, Waypoint.document_id == + DocumentGeometry.document_id) + if (needs_locale_join): - query = query.join(DocumentLocale, Waypoint.document_id == DocumentLocale.document_id) + query = query.join( + DocumentLocale, Waypoint.document_id == DocumentLocale.document_id) if (len(langs) > 0): query = query.filter(DocumentLocale.lang.in_(langs)) From 8a55a1aee1b398ddac0b859fa62d8cf5ad162da4 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Wed, 3 Dec 2025 11:40:16 +0100 Subject: [PATCH 26/41] [CI] don't create dir if exists --- launch_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_ci.sh b/launch_ci.sh index 994f4a375..e77bbe82a 100755 --- a/launch_ci.sh +++ b/launch_ci.sh @@ -5,7 +5,7 @@ apt update apt install -y postgresql-client cd /c2c_ci python -V -mkdir ~/.venvs +mkdir -p ~/.venvs python -m venv ~/.venvs/ci source ~/.venvs/ci/bin/activate pip install --upgrade pip setuptools wheel From 9054d0566b2fab95d6ef9872be57083981d4814f Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Wed, 3 Dec 2025 14:20:13 +0100 Subject: [PATCH 27/41] [CI] clean after test and debug comment --- Jenkinsfile | 3 ++- launch_ci.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b19099134..caa217140 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,8 +35,9 @@ pipeline { catch (Exception e) { currentBuild.result = 'FAILURE' // Mark the build as failed error "CI failed: ${e.getMessage()}" + } finally { + sh "podman-compose ${env.PODMAN_ARGS} down" } - sh "podman-compose ${env.PODMAN_ARGS} down" } } } diff --git a/launch_ci.sh b/launch_ci.sh index e77bbe82a..dc833de68 100755 --- a/launch_ci.sh +++ b/launch_ci.sh @@ -21,4 +21,4 @@ USER=github scripts/create_user_db_test.sh make -f config/so.test template curl -v http://elasticsearch:9200 export $(cat .env | grep -v "^#" | xargs) -pytest --cov-report term --cov-report xml --cov=c2corg_api +pytest --cov-report term --cov-report xml --cov=c2corg_api # --log-level=DEBUG -v --trace -x From c67b0b3b4a7969b3ad3addeca4c86f9fadbff6f0 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 3 Dec 2025 14:27:00 +0100 Subject: [PATCH 28/41] [fix] fix CI --- c2corg_api/models/common/document_types.py | 4 ++-- c2corg_api/models/common/sortable_search_attributes.py | 2 +- c2corg_api/scripts/es/fill_index.py | 2 ++ c2corg_api/search/utils.py | 4 ++-- c2corg_api/views/route.py | 6 +++--- c2corg_api/views/waypoint.py | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/c2corg_api/models/common/document_types.py b/c2corg_api/models/common/document_types.py index 356674425..3425dc919 100644 --- a/c2corg_api/models/common/document_types.py +++ b/c2corg_api/models/common/document_types.py @@ -9,9 +9,9 @@ WAYPOINT_TYPE = 'w' BOOK_TYPE = 'b' XREPORT_TYPE = 'x' -COVERAGE_TYPE = 'z' +COVERAGE_TYPE = 'v' ALL = [ AREA_TYPE, ARTICLE_TYPE, IMAGE_TYPE, MAP_TYPE, OUTING_TYPE, ROUTE_TYPE, - USERPROFILE_TYPE, WAYPOINT_TYPE, BOOK_TYPE, XREPORT_TYPE, COVERAGE_TYPE + USERPROFILE_TYPE, WAYPOINT_TYPE, BOOK_TYPE, XREPORT_TYPE ] diff --git a/c2corg_api/models/common/sortable_search_attributes.py b/c2corg_api/models/common/sortable_search_attributes.py index 3f3fe3608..5b72aa0d7 100644 --- a/c2corg_api/models/common/sortable_search_attributes.py +++ b/c2corg_api/models/common/sortable_search_attributes.py @@ -350,7 +350,7 @@ } -sortable_search_attr_by_field = { +search_attr_by_field = { 'quality': sortable_quality_types, 'access_time': sortable_access_times, 'paragliding_rating': sortable_paragliding_ratings, diff --git a/c2corg_api/scripts/es/fill_index.py b/c2corg_api/scripts/es/fill_index.py index d69ee0d0f..39d0a2e74 100644 --- a/c2corg_api/scripts/es/fill_index.py +++ b/c2corg_api/scripts/es/fill_index.py @@ -76,6 +76,8 @@ def progress(count, total_count): count = 0 with batch: for doc_type in document_types: + if doc_type == 'z': + continue print('Importing document type {}'.format(doc_type)) to_search_document = search_documents[doc_type].to_search_document diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index 540f4108f..9f2824f41 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -50,7 +50,7 @@ def build_sqlalchemy_filters( # the Geometry model (where ce access to geometry) geometry_model, # most likely always DocumentGeometry # the mapper for range enum - range_enum_map, # most likely always sortable_search_attr_by_field + range_enum_map, # most likely always search_attr_by_field # the column for the title # (ex: Waypoint -> title, Route -> title and title_prefix) title_columns=None @@ -73,7 +73,7 @@ def build_sqlalchemy_filters( document_model=Waypoint, filter_map={"areas": Area,}, geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, + range_enum_map=search_attr_by_field, title_columns=[DocumentLocale.title] ) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 7e5706fa7..7b90ea34a 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -45,7 +45,7 @@ from c2corg_api.models.area import schema_listing_area from sqlalchemy import func, literal_column from c2corg_api.models.common.sortable_search_attributes import \ - sortable_search_attr_by_field + search_attr_by_field validate_route_create = make_validator_create( @@ -610,7 +610,7 @@ def build_reachable_route_query(params, meta_params): document_model=Route, filter_map={'areas': Area, 'waypoints': Waypoint}, geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, + range_enum_map=search_attr_by_field, title_columns=[DocumentLocale.title, RouteLocale.title_prefix] ) @@ -681,7 +681,7 @@ def build_reachable_route_query_with_waypoints(params, meta_params): document_model=Route, filter_map={'areas': Area, 'waypoints': Waypoint}, geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, + range_enum_map=search_attr_by_field, title_columns=[DocumentLocale.title, RouteLocale.title_prefix] ) diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index 0ddb83a9c..b0da9d6e5 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -49,7 +49,7 @@ from c2corg_api.models.area import schema_listing_area from sqlalchemy import func from c2corg_api.models.common.sortable_search_attributes import \ - sortable_search_attr_by_field + search_attr_by_field log = logging.getLogger(__name__) @@ -560,7 +560,7 @@ def build_reachable_waypoints_query(params, meta_params): document_model=Waypoint, filter_map={"areas": Area}, geometry_model=DocumentGeometry, - range_enum_map=sortable_search_attr_by_field, + range_enum_map=search_attr_by_field, title_columns=[DocumentLocale.title] ) From e2b423e9d083ffe6611236af369838a953b2b6e5 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 3 Dec 2025 14:37:43 +0100 Subject: [PATCH 29/41] [fix] Fix CI --- c2corg_api/scripts/es/fill_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c2corg_api/scripts/es/fill_index.py b/c2corg_api/scripts/es/fill_index.py index 39d0a2e74..afd2778a9 100644 --- a/c2corg_api/scripts/es/fill_index.py +++ b/c2corg_api/scripts/es/fill_index.py @@ -76,7 +76,7 @@ def progress(count, total_count): count = 0 with batch: for doc_type in document_types: - if doc_type == 'z': + if doc_type == 'v': continue print('Importing document type {}'.format(doc_type)) to_search_document = search_documents[doc_type].to_search_document From 10057e6938901b330bf63837c20a861334d6f349 Mon Sep 17 00:00:00 2001 From: Leo Gourdin Date: Wed, 10 Dec 2025 09:23:57 +0100 Subject: [PATCH 30/41] [cleanup] remove S/O files before merging with upstream --- Jenkinsfile | 56 ------------------------------------------- config/so.test | 13 ---------- launch_ci.sh | 24 ------------------- podman-compose.ci.yml | 26 -------------------- 4 files changed, 119 deletions(-) delete mode 100644 Jenkinsfile delete mode 100644 config/so.test delete mode 100755 launch_ci.sh delete mode 100644 podman-compose.ci.yml diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index caa217140..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,56 +0,0 @@ -pipeline { - agent { - label 'ansible-agent' - } - - parameters { - // Define boolean variables for project deployment - user input - booleanParam(name: 'LaunchCI', defaultValue: true, description: 'Launch CI tests') - } - - environment { - // Static vars - REPO_SO_BASE_URL = 'git@git.smart-origin.com:SmartOrigin' - REPO_BACKEND_NAME = 'c2c_v6_api' - WORKDIR = 'tmp_c2c_ci' - PODMAN_ARGS = '-p c2c_ci -f podman-compose.ci.yml' - } - - stages { - stage('Launch CI tests') { - //The when block control is this stage should be run base on the params given - when { - expression { - params.LaunchCI == true - } - } - steps { - script { - sh """ - pwd - podman-compose ${env.PODMAN_ARGS} up -d - sleep 60 - """ - try { sh "podman-compose ${env.PODMAN_ARGS} exec test /c2c_ci/launch_ci.sh" } - catch (Exception e) { - currentBuild.result = 'FAILURE' // Mark the build as failed - error "CI failed: ${e.getMessage()}" - } finally { - sh "podman-compose ${env.PODMAN_ARGS} down" - } - } - } - } - } - post { - // Your post-build actions here - failure { - // Actions to take when any step or stage fails - echo 'The build has failed. Take a look at the logs and try again' - } - success { - // Actions to take when any step or stage fails - echo 'The build has succeded. Enjoy!' - } - } -} diff --git a/config/so.test b/config/so.test deleted file mode 100644 index 9057fa2bb..000000000 --- a/config/so.test +++ /dev/null @@ -1,13 +0,0 @@ -include config/dev - -export instanceid = github -export base_url = /${instanceid} -export db_name = c2corg_github_tests -export tests_db_name = c2corg_github_tests -export tests_db_host = postgresql -export elasticsearch_port = 9200 -export elasticsearch_index = c2corg_github_tests -export tests_elasticsearch_host = elasticsearch -export tests_elasticsearch_port = 9200 -export tests_elasticsearch_index = c2corg_github_tests -export redis_url = redis://redis:6379/ diff --git a/launch_ci.sh b/launch_ci.sh deleted file mode 100755 index dc833de68..000000000 --- a/launch_ci.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -e - -apt update -apt install -y postgresql-client -cd /c2c_ci -python -V -mkdir -p ~/.venvs -python -m venv ~/.venvs/ci -source ~/.venvs/ci/bin/activate -pip install --upgrade pip setuptools wheel -pip install dotenv flake8 -pip install -r dev-requirements.txt -r requirements.txt -flake8 c2corg_api es_migration -export PGHOST=postgresql -export PGPORT=5432 -export PGUSER=postgres -export PGPASSWORD=test -echo "create user \"www-data\" with password 'www-data'" | psql -USER=github scripts/create_user_db_test.sh -make -f config/so.test template -curl -v http://elasticsearch:9200 -export $(cat .env | grep -v "^#" | xargs) -pytest --cov-report term --cov-report xml --cov=c2corg_api # --log-level=DEBUG -v --trace -x diff --git a/podman-compose.ci.yml b/podman-compose.ci.yml deleted file mode 100644 index 011286a0a..000000000 --- a/podman-compose.ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -services: - postgresql: - image: postgis/postgis - restart: "no" - environment: - POSTGRES_USER: 'postgres' - POSTGRES_PASSWORD: 'test' - ports: - - 5432:5432 - - redis: - image: redis - restart: "no" - ports: - - 6379:6379 - - elasticsearch: - image: elasticsearch:2.4.6-alpine - restart: "no" - - test: - image: python:3.9-bookworm - restart: "no" - volumes: - - ./:/c2c_ci - entrypoint: ["/usr/bin/sleep", "30m"] From 3bb8456ffb02d29f02c2689e0c789cee57edeee5 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Thu, 4 Dec 2025 16:07:39 +0100 Subject: [PATCH 31/41] [fix] fix #1834 : durations filter wasn't working --- c2corg_api/search/utils.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index 9f2824f41..27d967abf 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -152,7 +152,10 @@ def build_sqlalchemy_filters( if filter_key == "range": filter_conditions.append( build_range_expression( - col, param_value, range_enum_map.get(param_key) + col, + param_value, + range_enum_map.get(param_key), + col_type ) ) @@ -185,7 +188,7 @@ def build_sqlalchemy_filters( return final_filter, sort_expressions, needs_locale_join, langs -def build_range_expression(col, param_value, enum_map): +def build_range_expression(col, param_value, enum_map, col_type): """ build sql alchemy filter for range expressions """ @@ -200,15 +203,22 @@ def build_range_expression(col, param_value, enum_map): values = [val for val, num in enum_map.items() if num == gte] else: values = [val for val, num in enum_map.items() if num >= - gte and num < lte] + gte and num <= lte] elif gte is not None: values = [val for val, num in enum_map.items() if num >= gte] elif lte is not None: - values = [val for val, num in enum_map.items() if num < lte] + values = [val for val, num in enum_map.items() if num <= lte] + + # if col type is an array of enum + if isinstance(col_type, ArrayOfEnum): + checks = [col.any(v) for v in values] + else: + checks = [col == v for v in values] - checks = [col == v for v in values] if not checks: return False + + # build OR by folding with | or_expr = checks[0] for check in checks[1:]: or_expr = or_expr | check From 3b999d39fabd14035a33935debc20c51597587c3 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Thu, 4 Dec 2025 16:08:28 +0100 Subject: [PATCH 32/41] [improve] add range filter to reduce time of computation --- c2corg_api/views/navitia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index 0860e24ec..5ffdef886 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -450,7 +450,7 @@ def post(self): self.request.body)['geom_detail'])) query = ( - DBSession.query(Area) + DBSession.query(Area).filter(Area.area_type == 'range') ) results = query.all() From 0ba34c32289fbd8e4157f8becf0d75ae85c914bc Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Fri, 5 Dec 2025 09:24:51 +0100 Subject: [PATCH 33/41] =?UTF-8?q?[fix]=20fix=20title=20query=20for=20waypo?= =?UTF-8?q?ints=20in=20Iitn=C3=A9vert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- c2corg_api/search/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index 27d967abf..cff8893e5 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -102,7 +102,10 @@ def build_sqlalchemy_filters( if query_value and title_columns: needs_locale_join = True like_clauses = [col.ilike(f"%{query_value}%") for col in title_columns] - filter_conditions.append(or_(*like_clauses)) + if len(like_clauses) == 2: + filter_conditions.append(or_(*like_clauses)) + elif len(like_clauses) == 1: + filter_conditions.append(like_clauses[0]) # loop over all elastic search filters for f in filters: From 3d6d24c18c5c74e1b04057fd9284d477e991af8a Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 8 Dec 2025 12:00:45 +0100 Subject: [PATCH 34/41] [fix] fix langs filter not working --- c2corg_api/views/route.py | 24 +++++++++++++----------- c2corg_api/views/waypoint.py | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 7b90ea34a..8ceca31c6 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -273,7 +273,7 @@ def get(self): json_areas.append(to_json_dict( area_obj, schema_listing_area)) - # assign JSON areas to the waypoint + # assign JSON areas to the route route.areas = json_areas wp_dict = to_json_dict(route, schema_route, True) routes.append(wp_dict) @@ -645,13 +645,14 @@ def build_reachable_route_query(params, meta_params): )). \ join(Area, Area.document_id == AreaAssociation.area_id) - if (needs_locale_join): - query = query. \ - join( + if (needs_locale_join or len(langs) > 0): + query = query.join( DocumentLocale, Route.document_id == DocumentLocale.document_id - ). \ - join(RouteLocale, RouteLocale.id == DocumentLocale.id) + ) + + if (needs_locale_join): + query = query.join(RouteLocale, RouteLocale.id == DocumentLocale.id) if (len(langs) > 0): query = query.filter(DocumentLocale.lang.in_(langs)) @@ -723,13 +724,14 @@ def build_reachable_route_query_with_waypoints(params, meta_params): )). \ join(Area, Area.document_id == AreaAssociation.area_id) - if (needs_locale_join): - query = query. \ - join( + if (needs_locale_join or len(langs) > 0): + query = query.join( DocumentLocale, Route.document_id == DocumentLocale.document_id - ). \ - join(RouteLocale, RouteLocale.id == DocumentLocale.id) + ) + + if (needs_locale_join): + query = query.join(RouteLocale, RouteLocale.id == DocumentLocale.id) if (len(langs) > 0): query = query.filter(DocumentLocale.lang.in_(langs)) diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index b0da9d6e5..782e3bf95 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -591,7 +591,7 @@ def build_reachable_waypoints_query(params, meta_params): join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id) - if (needs_locale_join): + if (needs_locale_join or len(langs) > 0): query = query.join( DocumentLocale, Waypoint.document_id == DocumentLocale.document_id) From 7dd7b2b6914937026f985a8b647a7bf70ab4103e Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 8 Dec 2025 14:41:54 +0100 Subject: [PATCH 35/41] [fix] fixed incorrect join with area association --- c2corg_api/views/route.py | 31 +++++++++++++++---------------- c2corg_api/views/waypoint.py | 8 ++++---- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 8ceca31c6..3762b7c98 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -634,15 +634,15 @@ def build_reachable_route_query(params, meta_params): join( WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id - ). \ + ). \ join( DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id - ). \ - join(AreaAssociation, or_( - AreaAssociation.document_id == Association.child_document_id, - AreaAssociation.document_id == Association.parent_document_id - )). \ + ). \ + join( + AreaAssociation, + AreaAssociation.document_id == Route.document_id + ). \ join(Area, Area.document_id == AreaAssociation.area_id) if (needs_locale_join or len(langs) > 0): @@ -718,20 +718,19 @@ def build_reachable_route_query_with_waypoints(params, meta_params): DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id ). \ - join(AreaAssociation, or_( - AreaAssociation.document_id == Association.child_document_id, - AreaAssociation.document_id == Association.parent_document_id - )). \ + join( + AreaAssociation, + AreaAssociation.document_id == Route.document_id + ). \ join(Area, Area.document_id == AreaAssociation.area_id) - if (needs_locale_join or len(langs) > 0): - query = query.join( + if (needs_locale_join): + query = query. \ + join( DocumentLocale, Route.document_id == DocumentLocale.document_id - ) - - if (needs_locale_join): - query = query.join(RouteLocale, RouteLocale.id == DocumentLocale.id) + ). \ + join(RouteLocale, RouteLocale.id == DocumentLocale.id) if (len(langs) > 0): query = query.filter(DocumentLocale.lang.in_(langs)) diff --git a/c2corg_api/views/waypoint.py b/c2corg_api/views/waypoint.py index 782e3bf95..8ec6fe163 100644 --- a/c2corg_api/views/waypoint.py +++ b/c2corg_api/views/waypoint.py @@ -583,10 +583,10 @@ def build_reachable_waypoints_query(params, meta_params): WaypointStoparea, WaypointStoparea.waypoint_id == Waypoint.document_id ). \ - join(AreaAssociation, or_( - AreaAssociation.document_id == Association.child_document_id, - AreaAssociation.document_id == Association.parent_document_id - )). \ + join( + AreaAssociation, + AreaAssociation.document_id == Waypoint.document_id + ). \ join(Area, Area.document_id == AreaAssociation.area_id). \ join(DocumentGeometry, Waypoint.document_id == DocumentGeometry.document_id) From 90bb53de239b023d2594c63715936e50c5f69c17 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Mon, 8 Dec 2025 14:42:07 +0100 Subject: [PATCH 36/41] [fix] fix title filter when no other filters --- c2corg_api/search/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/c2corg_api/search/utils.py b/c2corg_api/search/utils.py index cff8893e5..681e80c10 100644 --- a/c2corg_api/search/utils.py +++ b/c2corg_api/search/utils.py @@ -84,7 +84,10 @@ def build_sqlalchemy_filters( """ filters = search_dict.get("query", {}).get("bool", {}).get("filter", []) - must_list = search_dict.get("query", {}).get("bool", {}).get("must", []) + if len(filters) > 0: + must_list = search_dict.get("query", {}).get("bool", {}).get("must", []) # noqa + else: + must_list = [search_dict.get("query", {})] filter_conditions = [] needs_locale_join = False From 37e245d8eb16d37afcaeb500b3a5e8453af9c941 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Tue, 9 Dec 2025 15:57:04 +0100 Subject: [PATCH 37/41] [fix] fix lang filter --- c2corg_api/views/route.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/c2corg_api/views/route.py b/c2corg_api/views/route.py index 3762b7c98..3add9a03a 100644 --- a/c2corg_api/views/route.py +++ b/c2corg_api/views/route.py @@ -724,13 +724,14 @@ def build_reachable_route_query_with_waypoints(params, meta_params): ). \ join(Area, Area.document_id == AreaAssociation.area_id) - if (needs_locale_join): - query = query. \ - join( + if (needs_locale_join or len(langs) > 0): + query = query.join( DocumentLocale, Route.document_id == DocumentLocale.document_id - ). \ - join(RouteLocale, RouteLocale.id == DocumentLocale.id) + ) + + if (needs_locale_join): + query = query.join(RouteLocale, RouteLocale.id == DocumentLocale.id) if (len(langs) > 0): query = query.filter(DocumentLocale.lang.in_(langs)) From 7af82406e52483162ff4eed55c75ab3d694d93c1 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 10 Dec 2025 09:43:11 +0100 Subject: [PATCH 38/41] [clean] remove debug log --- c2corg_api/views/coverage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/c2corg_api/views/coverage.py b/c2corg_api/views/coverage.py index 9b44943d2..407edcd35 100644 --- a/c2corg_api/views/coverage.py +++ b/c2corg_api/views/coverage.py @@ -135,11 +135,8 @@ def get_coverages(polygon): # convert WKB → Shapely polygon poly = wkb_to_shape(geom) - log.warning(poly) - log.warning(polygon) if poly.contains(polygon) or poly.intersects(polygon): - log.warning("coverage found and added") coverage_found.append(coverage.coverage_type) return coverage_found From 499b96b7ed5b44a7b0fbbafc9dcd6f171bd248e4 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 10 Dec 2025 09:43:45 +0100 Subject: [PATCH 39/41] [new feature] add job handling for journey queries to monitor their progress --- c2corg_api/views/navitia.py | 628 ++++++++++++++++++++++++------------ 1 file changed, 422 insertions(+), 206 deletions(-) diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index 5ffdef886..c4ae91a44 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -2,6 +2,11 @@ import logging import os import requests +import redis +import uuid +import time +import threading +import ast from c2corg_api.models import DBSession from c2corg_api.models.area import Area from c2corg_api.models.utils import wkb_to_shape @@ -12,6 +17,7 @@ from c2corg_api.views.route import build_reachable_route_query_with_waypoints from shapely.geometry import Point from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError # noqa: E501 +from pyramid.response import Response from cornice.resource import resource, view from c2corg_api.views import cors_policy, to_json_dict from c2corg_api.models.route import schema_route @@ -19,6 +25,7 @@ from shapely.geometry import shape from pyproj import Transformer + log = logging.getLogger(__name__) # When editing these constants, make sure to edit them in the front too @@ -27,6 +34,11 @@ MAX_TRIP_DURATION = 240 MIN_TRIP_DURATION = 20 +# redis to store job's value (progress, result, error...) +REDIS_HOST = "redis" +REDIS_PORT = 6379 +REDIS_DB = 0 + def validate_navitia_params(request, **kwargs): """Validates the required parameters for the Navitia API""" @@ -143,145 +155,237 @@ def validate_journey_reachable_params(request, **kwargs): f'Paramètre {param} requis') -@resource(path='/navitia/journeyreachableroutes', cors_policy=cors_policy) -class NavitiaJourneyReachableRoutesRest: +@resource(path='/navitia/journeyreachableroutes/start', cors_policy=cors_policy) # noqa +class StartNavitiaJourneyReachableRoutesRest: def __init__(self, request, context=None): self.request = request @view(validators=[validate_journey_reachable_params]) def get(self): """ - Get all routes matching filters in params, that are reachable - (means there exists a Navitia journey for at least one of - their waypoints of type access). - - NOTE : the number of routes after applying filters, - has to be < MAX_ROUTE_THRESHOLD, - to reduce number of queries towards Navitia journey API + start job to retrieve journey reachable routes + returns job id """ - meta_params = extract_meta_params(self.request) + return start_job_background(computeJourneyReachableRoutes, self.request) # noqa - journey_params = extract_journey_params(self.request) - query = build_reachable_route_query_with_waypoints( - self.request.GET, meta_params) +@resource(path='/navitia/journeyreachablewaypoints/start', cors_policy=cors_policy) # noqa +class StartNavitiaJourneyReachableWaypointsRest: + def __init__(self, request, context=None): + self.request = request - results = ( - query.all() - ) + @view(validators=[validate_journey_reachable_params]) + def get(self): + """ + start job to retrieve journey reachable waypoints + returns job id + """ + return start_job_background(computeJourneyReachableWaypoints, self.request) # noqa - # /!\ IMPORTANT - # Before doing any further computations, - # make sure the number of routes don't go above threshold. - # The Itinevert UI doesn't allow that, - # but a user could query the api with wrong parameters... - if len(results) > MAX_ROUTE_THRESHOLD: - raise HTTPBadRequest( - "Couldn't proceed with computation : Too much routes found." - ) - areas_map = collect_areas_from_results(results, 1) +@resource(path='/navitia/journeyreachableroutes/result/{job_id}', cors_policy=cors_policy) # noqa +class NavitiaJourneyReachableRoutesResultRest: + def __init__(self, request, context=None): + self.request = request - wp_objects = collect_waypoints_from_results(results) + @view() + def get(self): + """ + get the result of the job : get journey reachable routes + returns the result + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return read_result_from_redis(r, job_id) - log.info("Number of NAVITIA journey queries : %d", len(wp_objects)) - navitia_wp_map = { - wp.document_id: is_wp_journey_reachable( - to_json_dict(wp, schema_waypoint), journey_params - ) for wp in wp_objects} +@resource(path='/navitia/journeyreachablewaypoints/result/{job_id}', cors_policy=cors_policy) # noqa +class NavitiaJourneyReachableWaypointsResultRest: + def __init__(self, request, context=None): + self.request = request - routes = [] - for route, areas, waypoints in results: - # check if a journey exists for route - # (at least one wp has a journey associated) - journey_exists = False - for wp in waypoints: - wp_id = wp.get("document_id") - journey_exists |= navitia_wp_map.get(wp_id) - - if journey_exists: - json_areas = [] - if areas is None: - areas = [] + @view() + def get(self): + """ + get the result of the job : get journey reachable waypoints + returns the result + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return read_result_from_redis(r, job_id) - for area in areas: - area_obj = areas_map.get(area.get("document_id")) - if area_obj: - json_areas.append(to_json_dict( - area_obj, schema_listing_area)) +# Progress endpoints - # assign JSON areas to the waypoint - route.areas = json_areas - wp_dict = to_json_dict(route, schema_route, True) - routes.append(wp_dict) - return {'documents': routes, 'total': len(routes)} +@resource(path='/navitia/journeyreachableroutes/progress/{job_id}', cors_policy=cors_policy) # noqa +class NavitiaJourneyReachableRoutesProgressRest: + def __init__(self, request, context=None): + self.request = request + + @view() + def get(self): + """ + monitor progress of job id for journey reachable routes + """ + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return Response(app_iter=progress_stream(r, job_id), content_type="text/event-stream") # noqa -@resource(path='/navitia/journeyreachablewaypoints', cors_policy=cors_policy) -class NavitiaJourneyReachableWaypointsRest: +@resource(path='/navitia/journeyreachablewaypoints/progress/{job_id}', cors_policy=cors_policy) # noqa +class NavitiaJourneyReachableWaypointsProgressRest: def __init__(self, request, context=None): self.request = request - @view(validators=[validate_journey_reachable_params]) + @view() def get(self): """ - Get all waypoints matching filters in params, that are reachable - (means there exists a Navitia journey). - NOTE : waypoints should be filtered with one area, - to reduce the number of queries towards Navitia journey API. + monitor progress of job id for journey reachable waypoints """ - meta_params = extract_meta_params(self.request) + r = redis_client() + job_id = self.request.matchdict.get("job_id") + return Response(app_iter=progress_stream(r, job_id), content_type="text/event-stream") # noqa + + +def computeJourneyReachableRoutes(job_id, request): + """ + Get all waypoints matching filters in params, that are reachable + (means there exists a Navitia journey for at least one of + their waypoints of type access). + + NOTE : the number of routes after applying filters, + has to be < MAX_ROUTE_THRESHOLD, + to reduce number of queries towards Navitia journey API + + the result can be found inside redis + """ + r = redis_client() + try: + meta_params = extract_meta_params(request) + journey_params = extract_journey_params(request) + query = build_reachable_route_query_with_waypoints( + request.GET, meta_params) + results = query.all() + + if len(results) > MAX_ROUTE_THRESHOLD: + raise HTTPBadRequest( + "Couldn't proceed with computation : Too much routes found.") + + areas_map = collect_areas_from_results(results, 1) + wp_objects = collect_waypoints_from_results(results) - journey_params = extract_journey_params(self.request) + total = len(wp_objects) + log.info("Number of NAVITIA journey queries : %d", total) + r.set(f"job:{job_id}:total", total) - areas = None + count = found = not_found = 0 + navitia_wp_map = {} + + for wp in wp_objects: + result = is_wp_journey_reachable( + to_json_dict(wp, schema_waypoint), journey_params) + navitia_wp_map[wp.document_id] = result + count += 1 + if result: + found += 1 + else: + not_found += 1 + _store_job_progress(r, job_id, count, found, not_found) + + routes = [] + for route, areas, waypoints in results: + journey_exists = any(navitia_wp_map.get( + wp.get("document_id")) for wp in waypoints) + if not journey_exists: + continue + json_areas = [] + for area in (areas or []): + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + route.areas = json_areas + routes.append(to_json_dict(route, schema_route, True)) + + r.set(f"job:{job_id}:result", json.dumps( + {'documents': routes, 'total': len(routes)})) + r.set(f"job:{job_id}:status", "done") + except Exception as exc: + log.exception(str(exc)) + r.set(f"job:{job_id}:status", "error") + r.set(f"job:{job_id}:error", str(exc)) + + +def computeJourneyReachableWaypoints(job_id, request): + """ + Get all routes matching filters in params, that are reachable + (means there exists a Navitia journey for at least one of + their waypoints of type access). + + NOTE : the waypoints have to be filtered by one area (and not more) + to reduce number of request towards Navitia Journey API + + the result can be found inside redis + """ + r = redis_client() + try: + meta_params = extract_meta_params(request) + journey_params = extract_journey_params(request) + + # Ensure areas filter is provided and normalized + areas_param = None try: - areas = self.request.GET['a'].split(",") + areas_param = request.GET['a'] + if isinstance(areas_param, str): + areas_list = areas_param.split(",") + else: + areas_list = list(areas_param) except Exception: - areas = None + areas_list = None - # Normalize: allow single value or list - if areas is None: + if areas_list is None: raise HTTPBadRequest('Missing filter : area is required') - elif isinstance(areas, list): - if len(areas) > 1: - raise HTTPBadRequest('Only one filtering area is allowed') + if len(areas_list) > 1: + raise HTTPBadRequest('Only one filtering area is allowed') - query = build_reachable_waypoints_query( - self.request.GET, meta_params) - - results = ( - query.all() - ) + query = build_reachable_waypoints_query(request.GET, meta_params) + results = query.all() areas_map = collect_areas_from_results(results, 1) - log.info("Number of NAVITIA journey queries : %d", len(results)) + total = len(results) + log.info("Number of NAVITIA journey queries : %d", total) + r.set(f"job:{job_id}:total", total) + count = found = not_found = 0 waypoints = [] + for waypoint, areas in results: - # check if a journey exists for waypoint - if is_wp_journey_reachable( - to_json_dict(waypoint, schema_waypoint), journey_params - ): + count += 1 + r.publish(f"job:{job_id}:events", f"not_found:{not_found}") + reachable = is_wp_journey_reachable(to_json_dict( + waypoint, schema_waypoint), journey_params) + if reachable: + found += 1 json_areas = [] - if areas is None: - areas = [] - - for area in areas: + for area in (areas or []): area_obj = areas_map.get(area.get("document_id")) if area_obj: json_areas.append(to_json_dict( area_obj, schema_listing_area)) - - # assign JSON areas to the waypoint waypoint.areas = json_areas - wp_dict = to_json_dict(waypoint, schema_waypoint, True) - waypoints.append(wp_dict) + waypoints.append(to_json_dict(waypoint, schema_waypoint, True)) + else: + not_found += 1 + _store_job_progress(r, job_id, count, found, not_found) - return {'documents': waypoints, 'total': len(waypoints)} + r.set(f"job:{job_id}:result", json.dumps( + {'documents': waypoints, 'total': len(waypoints)})) + r.set(f"job:{job_id}:status", "done") + except Exception as exc: + log.exception("Error computing reachable waypoints") + r.set(f"job:{job_id}:status", "error") + r.set(f"job:{job_id}:error", str(exc)) def validate_isochrone_reachable_params(request, **kwargs): @@ -311,62 +415,65 @@ def get(self): The isochron is created by querying navitia api with specific parameters, see validate_isochrone_reachable_params func """ - meta_params = extract_meta_params(self.request) + try: + meta_params = extract_meta_params(self.request) - isochrone_params = extract_isochrone_params(self.request) + isochrone_params = extract_isochrone_params(self.request) - query = build_reachable_route_query_with_waypoints( - self.request.GET, meta_params) + query = build_reachable_route_query_with_waypoints( + self.request.GET, meta_params) - results = query.all() + results = query.all() - areas_map = collect_areas_from_results(results, 1) + areas_map = collect_areas_from_results(results, 1) - response = get_navitia_isochrone(isochrone_params) + response = get_navitia_isochrone(isochrone_params) - routes = [] - geojson = "" - # if isochrone found - if (len(response["isochrones"]) > 0): - geojson = response["isochrones"][0]["geojson"] - isochrone_geom = shape(geojson) - - wp_objects = collect_waypoints_from_results(results) - - navitia_wp_map = {wp.document_id: is_wp_in_isochrone( - to_json_dict(wp, schema_waypoint), isochrone_geom - ) for wp in wp_objects} - - for route, areas, waypoints in results: - # check if a journey exists for route - # (at least one wp has a journey associated) - one_wp_in_isochrone = False - for wp in waypoints: - wp_id = wp.get("document_id") - one_wp_in_isochrone |= navitia_wp_map.get(wp_id) - - if one_wp_in_isochrone: - json_areas = [] - - if areas is None: - areas = [] - - for area in areas: - area_obj = areas_map.get(area.get("document_id")) - if area_obj: - json_areas.append(to_json_dict( - area_obj, schema_listing_area)) - - # assign JSON areas to the waypoint - route.areas = json_areas - route_dict = to_json_dict(route, schema_route, True) - routes.append(route_dict) - - return { - 'documents': routes, - 'total': len(routes), - 'isochron_geom': geojson - } + routes = [] + geojson = "" + # if isochrone found + if (len(response["isochrones"]) > 0): + geojson = response["isochrones"][0]["geojson"] + isochrone_geom = shape(geojson) + + wp_objects = collect_waypoints_from_results(results) + + navitia_wp_map = {wp.document_id: is_wp_in_isochrone( + to_json_dict(wp, schema_waypoint), isochrone_geom + ) for wp in wp_objects} + + for route, areas, waypoints in results: + # check if a journey exists for route + # (at least one wp has a journey associated) + one_wp_in_isochrone = False + for wp in waypoints: + wp_id = wp.get("document_id") + one_wp_in_isochrone |= navitia_wp_map.get(wp_id) + + if one_wp_in_isochrone: + json_areas = [] + + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + route.areas = json_areas + route_dict = to_json_dict(route, schema_route, True) + routes.append(route_dict) + + return { + 'documents': routes, + 'total': len(routes), + 'isochron_geom': geojson + } + except Exception as e: + return json.dumps(ast.literal_eval(str(e))) @resource( @@ -385,52 +492,56 @@ def get(self): The isochron is created by querying navitia api with specific parameters, see validate_isochrone_reachable_params func """ - meta_params = extract_meta_params(self.request) + try: - isochrone_params = extract_isochrone_params(self.request) + meta_params = extract_meta_params(self.request) - query = build_reachable_waypoints_query( - self.request.GET, meta_params) + isochrone_params = extract_isochrone_params(self.request) - results = query.all() + query = build_reachable_waypoints_query( + self.request.GET, meta_params) - # manage areas for waypoints - areas_map = collect_areas_from_results(results, 1) + results = query.all() - response = get_navitia_isochrone(isochrone_params) + # manage areas for waypoints + areas_map = collect_areas_from_results(results, 1) - waypoints = [] - geojson = "" - # if isochrone found - if (len(response["isochrones"]) > 0): - geojson = response["isochrones"][0]["geojson"] - isochrone_geom = shape(geojson) - - for waypoint, areas in results: - # check if wp is in isochrone - if is_wp_in_isochrone( - to_json_dict(waypoint, schema_waypoint), isochrone_geom - ): - json_areas = [] - if areas is None: - areas = [] - - for area in areas: - area_obj = areas_map.get(area.get("document_id")) - if area_obj: - json_areas.append(to_json_dict( - area_obj, schema_listing_area)) - - # assign JSON areas to the waypoint - waypoint.areas = json_areas - wp_dict = to_json_dict(waypoint, schema_waypoint, True) - waypoints.append(wp_dict) - - return { - 'documents': waypoints, - 'total': len(waypoints), - "isochron_geom": geojson - } + response = get_navitia_isochrone(isochrone_params) + + waypoints = [] + geojson = "" + # if isochrone found + if (len(response["isochrones"]) > 0): + geojson = response["isochrones"][0]["geojson"] + isochrone_geom = shape(geojson) + + for waypoint, areas in results: + # check if wp is in isochrone + if is_wp_in_isochrone( + to_json_dict(waypoint, schema_waypoint), isochrone_geom + ): + json_areas = [] + if areas is None: + areas = [] + + for area in areas: + area_obj = areas_map.get(area.get("document_id")) + if area_obj: + json_areas.append(to_json_dict( + area_obj, schema_listing_area)) + + # assign JSON areas to the waypoint + waypoint.areas = json_areas + wp_dict = to_json_dict(waypoint, schema_waypoint, True) + waypoints.append(wp_dict) + + return { + 'documents': waypoints, + 'total': len(waypoints), + "isochron_geom": geojson + } + except Exception as e: + return json.dumps(ast.literal_eval(str(e))) @resource(path='/navitia/areainisochrone', cors_policy=cors_policy) @@ -488,7 +599,8 @@ def is_wp_journey_reachable(waypoint, journey_params): # Récupération de la clé API depuis les variables d'environnement api_key = os.getenv('NAVITIA_API_KEY') if not api_key: - return False + raise HTTPInternalServerError( + 'Configuration API Navitia manquante') response = {} @@ -511,30 +623,39 @@ def is_wp_journey_reachable(waypoint, journey_params): # Vérification du statut de la réponse if response.status_code == 401: - return False + raise HTTPInternalServerError('Authentication error with Navitia API') # noqa elif response.status_code == 400: - return False + raise HTTPBadRequest('Invalid parameters for Navitia API') elif response.status_code == 404: + # no_destination -> public transport not reachable from destination + # no_origin -> public transport not reachable from origin + # these do not count as proper errors, + # more like the wp is just not reachable + if response.json()['error']['id'] != 'no_destination' and \ + response.json()['error']['id'] != 'no_origin': + raise HTTPInternalServerError(response.json()['error']) return False elif not response.ok: - return False - - # make sure the waypoint is reachable if at least one journey's - # departure date time is the same day as the day in journey_params - for journey in response.json()['journeys']: - journey_day = int(journey['departure_date_time'][6:8]) - param_day = int(journey_params['datetime'][6:8]) - if journey_day == param_day: - return True + raise HTTPInternalServerError(f'Navitia API error: {response.status_code}') # noqa: E501 + else: + # code 200 OK + # make sure the waypoint is reachable if at least one journey's + # departure date time is the same day as the day in journey_params + for journey in response.json().get('journeys', []): + journey_day = int(journey['departure_date_time'][6:8]) + param_day = int(journey_params['datetime'][6:8]) + if journey_day == param_day: + return True - return False + return False except requests.exceptions.Timeout: - return False - except requests.exceptions.RequestException: - return False - except Exception: - return False + raise HTTPInternalServerError( + 'Timeout when calling the Navitia API') + except requests.exceptions.RequestException as e: + raise HTTPInternalServerError(f'{str(e)}') + except Exception as e: + raise HTTPInternalServerError(f'{str(e)}') def get_navitia_isochrone(isochrone_params): @@ -574,20 +695,24 @@ def get_navitia_isochrone(isochrone_params): elif response.status_code == 400: raise HTTPBadRequest('Invalid parameters for Navitia API') elif response.status_code == 404: - return {} + # no_destination -> public transport not reachable from destination + # no_origin -> public transport not reachable from origin + # these do not count as proper errors, + # more like the wp is just not reachable + raise HTTPInternalServerError(response.json()['error']) elif not response.ok: raise HTTPInternalServerError(f'Navitia API error: {response.status_code}') # noqa: E501 - - # Retour des données JSON - return response.json() + else: + # Retour des données JSON + return response.json() except requests.exceptions.Timeout: raise HTTPInternalServerError( 'Timeout when calling the Navitia API') except requests.exceptions.RequestException as e: - raise HTTPInternalServerError(f'Network error: {str(e)}') + raise HTTPInternalServerError(f'{str(e)}') except Exception as e: - raise HTTPInternalServerError(f'Internal error: {str(e)}') + raise HTTPInternalServerError(f'{str(e)}') def is_wp_in_isochrone(waypoint, isochrone_geom): @@ -708,3 +833,94 @@ def collect_waypoints_from_results(results): ).all() return {wp for wp in wp_objects} + + +def redis_client(): + """ fast way to get redis client """ + return redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB) + + +def start_job_background(target, request): + """ start a job in the background, + target is the query function to execute in bg + request is the request to pass to the function""" + job_id = str(uuid.uuid4()) + r = redis_client() + r.set(f"job:{job_id}:progress", 0) + r.set(f"job:{job_id}:status", "running") + threading.Thread(target=target, args=( + job_id, request), daemon=True).start() + return {"job_id": job_id} + + +def get_job_status(r, job_id): + """ returns the ongoing job status """ + status = r.get(f"job:{job_id}:status") + if status is None: + return None, {"error": "unknown_job_id"} + status = status.decode() + return status, status + + +def read_result_from_redis(r, job_id): + """ returns the result from redis """ + status = r.get(f"job:{job_id}:status") + if status is None: + return {"error": "unknown_job_id"} + status = status.decode() + if status == "running": + return {"status": "running"} + if status == "error": + error_msg = r.get(f"job:{job_id}:error") + return {"status": "error", "message": error_msg.decode() if error_msg else "unknown error"} # noqa + if status == "done": + data = r.get(f"job:{job_id}:result") + if not data: + return {"status": "error", "message": "missing_result"} + return {"status": "done", "result": json.loads(data)} + return {"error": "unknown_status", "status": status} + + +def progress_stream(r, job_id, poll_interval=0.5): + """ yield the job progress """ + while True: + raw_progress = r.get(f"job:{job_id}:progress") + raw_found = r.get(f"job:{job_id}:found") + raw_not_found = r.get(f"job:{job_id}:not_found") + raw_total = r.get(f"job:{job_id}:total") + + progress = int(raw_progress) if raw_progress is not None else 0 + found = int(raw_found) if raw_found is not None else 0 + not_found = int(raw_not_found) if raw_not_found is not None else 0 + total = int(raw_total) if raw_total is not None else 0 + + payload = {"progress": progress, "total": total, + "found": found, "not_found": not_found} + yield (f"data: {json.dumps(payload)}\n\n").encode("utf-8") + + status = r.get(f"job:{job_id}:status") + if status and status.decode() == "done": + yield b"event: done\ndata: done\n\n" + break + elif status and status.decode() == "error": + payload = r.get(f"job:{job_id}:error") + json_payload = json.dumps(ast.literal_eval(payload.decode())) + yield f"event: error\ndata: {json_payload}\n\n".encode("utf-8") + break + + time.sleep(poll_interval) + + +def _store_job_progress(r, job_id, count, found, not_found): + """ + store job progress which is : + progress : the number of queries done + found : the number of successful queries + not_found: the number of unsuccessful queries + """ + r.set(f"job:{job_id}:progress", count) + r.set(f"job:{job_id}:found", found) + r.set(f"job:{job_id}:not_found", not_found) + r.publish(f"job:{job_id}:events", f"progress:{count}") + r.publish(f"job:{job_id}:events", f"found:{found}") + r.publish(f"job:{job_id}:events", f"not_found:{not_found}") From 53ff11f3afe491e680d96cd3304fe4e921a22102 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 10 Dec 2025 14:56:30 +0100 Subject: [PATCH 40/41] [lint] fix linter -> func names in lower case --- c2corg_api/views/navitia.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/c2corg_api/views/navitia.py b/c2corg_api/views/navitia.py index c4ae91a44..692cc7c87 100644 --- a/c2corg_api/views/navitia.py +++ b/c2corg_api/views/navitia.py @@ -166,7 +166,7 @@ def get(self): start job to retrieve journey reachable routes returns job id """ - return start_job_background(computeJourneyReachableRoutes, self.request) # noqa + return start_job_background(compute_journey_reachable_routes, self.request) # noqa @resource(path='/navitia/journeyreachablewaypoints/start', cors_policy=cors_policy) # noqa @@ -180,7 +180,7 @@ def get(self): start job to retrieve journey reachable waypoints returns job id """ - return start_job_background(computeJourneyReachableWaypoints, self.request) # noqa + return start_job_background(compute_journey_reachable_waypoints, self.request) # noqa @resource(path='/navitia/journeyreachableroutes/result/{job_id}', cors_policy=cors_policy) # noqa @@ -247,7 +247,7 @@ def get(self): return Response(app_iter=progress_stream(r, job_id), content_type="text/event-stream") # noqa -def computeJourneyReachableRoutes(job_id, request): +def compute_journey_reachable_routes(job_id, request): """ Get all waypoints matching filters in params, that are reachable (means there exists a Navitia journey for at least one of @@ -316,7 +316,7 @@ def computeJourneyReachableRoutes(job_id, request): r.set(f"job:{job_id}:error", str(exc)) -def computeJourneyReachableWaypoints(job_id, request): +def compute_journey_reachable_waypoints(job_id, request): """ Get all routes matching filters in params, that are reachable (means there exists a Navitia journey for at least one of From 2f5d6cbef9c7a6566f56336ea139a296892ee0b6 Mon Sep 17 00:00:00 2001 From: "gerome.perrin" Date: Wed, 10 Dec 2025 15:11:46 +0100 Subject: [PATCH 41/41] [fix] fix search doc missing for coverage --- c2corg_api/scripts/es/fill_index.py | 2 -- c2corg_api/search/__init__.py | 5 +++- .../search/mappings/coverage_mapping.py | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 c2corg_api/search/mappings/coverage_mapping.py diff --git a/c2corg_api/scripts/es/fill_index.py b/c2corg_api/scripts/es/fill_index.py index afd2778a9..d69ee0d0f 100644 --- a/c2corg_api/scripts/es/fill_index.py +++ b/c2corg_api/scripts/es/fill_index.py @@ -76,8 +76,6 @@ def progress(count, total_count): count = 0 with batch: for doc_type in document_types: - if doc_type == 'v': - continue print('Importing document type {}'.format(doc_type)) to_search_document = search_documents[doc_type].to_search_document diff --git a/c2corg_api/search/__init__.py b/c2corg_api/search/__init__.py index 8b2186bc6..7b81cf5d1 100644 --- a/c2corg_api/search/__init__.py +++ b/c2corg_api/search/__init__.py @@ -9,6 +9,7 @@ from c2corg_api.models.user_profile import USERPROFILE_TYPE from c2corg_api.models.waypoint import WAYPOINT_TYPE from c2corg_api.models.xreport import XREPORT_TYPE +from c2corg_api.models.coverage import COVERAGE_TYPE from c2corg_api.search.mappings.area_mapping import SearchArea from c2corg_api.search.mappings.article_mapping import SearchArticle from c2corg_api.search.mappings.book_mapping import SearchBook @@ -19,6 +20,7 @@ from c2corg_api.search.mappings.user_mapping import SearchUser from c2corg_api.search.mappings.waypoint_mapping import SearchWaypoint from c2corg_api.search.mappings.xreport_mapping import SearchXreport +from c2corg_api.search.mappings.coverage_mapping import SearchCoverage from elasticsearch import Elasticsearch from elasticsearch_dsl import Search from elasticsearch_dsl.connections import connections @@ -114,5 +116,6 @@ def get_text_query_on_title(search_term, search_lang=None): ROUTE_TYPE: SearchRoute, MAP_TYPE: SearchTopoMap, USERPROFILE_TYPE: SearchUser, - WAYPOINT_TYPE: SearchWaypoint + WAYPOINT_TYPE: SearchWaypoint, + COVERAGE_TYPE: SearchCoverage, } diff --git a/c2corg_api/search/mappings/coverage_mapping.py b/c2corg_api/search/mappings/coverage_mapping.py new file mode 100644 index 000000000..2e3a1af04 --- /dev/null +++ b/c2corg_api/search/mappings/coverage_mapping.py @@ -0,0 +1,29 @@ +from c2corg_api.models.coverage import COVERAGE_TYPE, Coverage +from c2corg_api.search.mapping import SearchDocument, BaseMeta +from c2corg_api.search.mapping_types import QueryableMixin, QEnum + + +class SearchCoverage(SearchDocument): + class Meta(BaseMeta): + doc_type = COVERAGE_TYPE + + coverage_type = QEnum('ctyp', model_field=Coverage.coverage_type) + + FIELDS = ['coverage_type'] + + @staticmethod + def to_search_document(document, index): + search_document = SearchDocument.to_search_document( + document, index, include_areas=False) + + if document.redirects_to: + return search_document + + SearchDocument.copy_fields( + search_document, document, SearchCoverage.FIELDS) + + return search_document + + +SearchCoverage.queryable_fields = \ + QueryableMixin.get_queryable_fields(SearchCoverage)